Skip to content

Instantly share code, notes, and snippets.

@pashu123

pashu123/vec_1_16_16.mlir

Created April 23, 2024 08:53
Show Gist options
  • Save pashu123/0ba9090307ee18b34a693328be1727be to your computer and use it in GitHub Desktop.
Save pashu123/0ba9090307ee18b34a693328be1727be to your computer and use it in GitHub Desktop.
Download ZIP
Raw
vec_1_16_16.mlir
This file has been truncated, but you can view the full file.
// -----// IR Dump After AssignTargetDevicesPass (iree-hal-assign-target-devices) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
func.func @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After AutoInputConversionPipeline (iree-auto-input-conversion) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
func.func @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After IREEImportPublic (iree-import-public) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After ImportMLProgram (iree-import-ml-program) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After SanitizeModuleNames (iree-sanitize-module-names) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After ConvertMeshToFlow (iree-convert-mesh-to-flow) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After mlir::iree_compiler::IREE::ABI::ConvertStreamableOpsPass (iree-abi-convert-streamable-ops) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After mlir::iree_compiler::IREE::ABI::WrapEntryPointsPass (iree-abi-wrap-entry-points) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[1] : index
%2 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<?x?x3200xf32>{%0, %1}
%3 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%4 = util.call @_broadcast_pack_kernel(%2, %3) : (tensor<?x?x3200xf32>, tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16>
%c0 = arith.constant 0 : index
%dim = tensor.dim %4, %c0 : tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %4 "output0" : tensor<?x540x3200x16x1xf16>{%dim} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
util.func private @_broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func private @_broadcast_pack_kernel(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16> {
%c0 = arith.constant 0 : index
%dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32>
%0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16>
%1 = tensor.empty(%dim) : tensor<?x8640x3200xf16>
%2 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %2 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %0 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
util.return %pack : tensor<?x540x3200x16x1xf16>
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[1] : index
%2 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<?x?x3200xf32>{%0, %1}
%3 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%4 = util.call @_broadcast_pack_kernel(%2, %3) : (tensor<?x?x3200xf32>, tensor<8640x3200xf16>) -> tensor<?x540x3200x16x1xf16>
%dim = tensor.dim %4, %c0 : tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %4 "output0" : tensor<?x540x3200x16x1xf16>{%dim} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Inliner (inline) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SymbolDCE (symbol-dce) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After DemoteF64ToF32 (iree-util-demote-f64-to-f32) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After RemoveZeroExtentTensors (iree-global-opt-remove-zero-extent-tensors) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After DetachElementwiseFromNamedOps (iree-global-opt-detach-elementwise-from-named-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After LinalgNamedOpConversionPass (linalg-named-op-conversion) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Convert1X1FilterConv2DToMatmul (iree-global-opt-convert-1x1-filter-conv2d-to-matmul) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After EraseUnusedLinalgOperands (iree-global-opt-erase-unused-linalg-operands) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After ExpandTensorShapes (iree-global-opt-expand-tensor-shapes) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After ConvertElementwiseToLinalgPass (convert-elementwise-to-linalg) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After RaiseSpecialOps (iree-global-opt-raise-special-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After DecomposeConcat (iree-global-opt-decompose-concat) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After GeneralizeLinalgNamedOps (iree-global-opt-generalize-linalg-named-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After FoldUnitExtentDimsPass (iree-flow-fold-unit-extent-dims) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After FuseDequantizationMatmul (iree-global-opt-fuse-dequantization-matmul) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SetEncoding (iree-global-opt-set-encoding) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After CPUMaterializeUpperBoundTileSize (iree-codegen-cpu-materialize-upper-bound-tile-size) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CPUMaterializeEncoding (iree-codegen-cpu-materialize-encoding) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After MaterializeHomogeneousEncodings (iree-global-opt-materialize-homogeneous-encodings) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After CSE (cse) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After SimplifyPackUnpack (iree-global-opt-simplify-pack-unpack) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After DataLayoutPropagation (iree-global-opt-data-layout-propagation) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After GeneralizeLinalgNamedOps (iree-global-opt-generalize-linalg-named-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After GlobalLoopInvariantCodeMotion (iree-global-opt-loop-invariant-code-motion) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After CSE (cse) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After HoistIntoGlobals (iree-util-hoist-into-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After JitGlobals (iree-consteval-jit-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After RaiseSpecialOps (iree-global-opt-raise-special-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After VerifyInputLegalityPass (iree-verify-input-legality) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After InjectTensorTracingPass (iree-flow-inject-tensor-tracing) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After TensorPadToTensorInsertSlicePass (iree-flow-tensor-pad-to-tensor-insert-slice) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After InterchangeGenericOpsPass (iree-flow-interchange-generic-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ResolveShapedTypeResultDims (resolve-shaped-type-result-dims) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ElementwiseOpFusionPass (iree-flow-elementwise-op-fusion) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After BubbleUpExpandShapesPass (iree-flow-bubble-up-expand-shapes) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ElementwiseOpFusionPass (iree-flow-elementwise-op-fusion) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After FusionOfTensorOpsPass (iree-flow-fusion-of-tensor-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SplitReductionPass (iree-flow-split-reduction-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After InterchangeGenericOpsPass (iree-flow-interchange-generic-ops) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After FormScalarDispatchesPass (iree-flow-form-scalar-dispatches) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %4 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
%5 = hal.tensor.export %pack "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After FormDispatchRegionsPass (iree-flow-form-dispatch-regions) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%c0 = arith.constant 0 : index
%4 = flow.dispatch.region -> (tensor<?x540x3200x16x1xf16>{%0}) {
%6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %2 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.return %pack : tensor<?x540x3200x16x1xf16>
}
%5 = hal.tensor.export %4 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CloneProducersIntoDispatchRegionsPass (iree-flow-clone-producers-into-dispatch-regions) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%c0 = arith.constant 0 : index
%4 = flow.dispatch.region -> (tensor<?x540x3200x16x1xf16>{%0}) {
%6 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%7 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %8 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %6 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.return %pack : tensor<?x540x3200x16x1xf16>
}
%5 = hal.tensor.export %4 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CollapseDimensionsPass (iree-flow-collapse-dimensions) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%3 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%c0 = arith.constant 0 : index
%4 = flow.dispatch.region -> (tensor<?x540x3200x16x1xf16>{%0}) {
%6 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%7 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %8 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %6 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.return %pack : tensor<?x540x3200x16x1xf16>
}
%5 = hal.tensor.export %4 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After FormDispatchWorkgroupsPass (iree-flow-form-dispatch-workgroups) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0} =
(%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%4 = flow.dispatch.workload.ordinal %arg3, 0 : index
%5 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%6 = tensor.empty(%4) : tensor<?x540x3200x16x1xf16>
%7 = tensor.empty(%4) : tensor<?x8640x3200xf16>
%8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%5 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %8 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %6 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %arg4, offsets = [0, 0, 0, 0, 0], sizes = [%4, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
flow.return
} count(%arg2: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2
flow.return %x, %y, %z : index, index, index
}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After CaptureDynamicDimsPass (iree-flow-capture-dynamic-dims) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0} =
(%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%4 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%arg3}
%5 = flow.dispatch.workload.ordinal %arg3, 0 : index
%6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%7 = tensor.empty(%5) : tensor<?x540x3200x16x1xf16>
%8 = tensor.empty(%5) : tensor<?x8640x3200xf16>
%9 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%8 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %9 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %7 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %4, offsets = [0, 0, 0, 0, 0], sizes = [%5, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%5}
flow.return
} count(%arg2: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2
flow.return %x, %y, %z : index, index, index
}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0} =
(%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%4 = flow.dispatch.workload.ordinal %arg3, 0 : index
%5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
%6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%7 = tensor.empty(%4) : tensor<?x540x3200x16x1xf16>
%8 = tensor.empty(%4) : tensor<?x8640x3200xf16>
%9 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%8 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %9 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %7 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %5, offsets = [0, 0, 0, 0, 0], sizes = [%4, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
flow.return
} count(%arg2: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2
flow.return %x, %y, %z : index, index, index
}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0} =
(%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%4 = flow.dispatch.workload.ordinal %arg3, 0 : index
%5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
%6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%7 = tensor.empty(%4) : tensor<?x540x3200x16x1xf16>
%8 = tensor.empty(%4) : tensor<?x8640x3200xf16>
%9 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%8 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %9 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %7 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %5, offsets = [0, 0, 0, 0, 0], sizes = [%4, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
flow.return
} count(%arg2: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2
flow.return %x, %y, %z : index, index, index
}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After InitializeEmptyTensorsPass (iree-flow-initialize-empty-tensors) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0} =
(%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%4 = flow.dispatch.workload.ordinal %arg3, 0 : index
%5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
%6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%7 = tensor.empty(%4) : tensor<?x540x3200x16x1xf16>
%8 = tensor.empty(%4) : tensor<?x8640x3200xf16>
%9 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%8 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %9 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %7 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %5, offsets = [0, 0, 0, 0, 0], sizes = [%4, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
flow.return
} count(%arg2: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2
flow.return %x, %y, %z : index, index, index
}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After OutlineDispatchExternsPass (iree-flow-outline-dispatch-externs) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0} =
(%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%4 = flow.dispatch.workload.ordinal %arg3, 0 : index
%5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
%6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%7 = tensor.empty(%4) : tensor<?x540x3200x16x1xf16>
%8 = tensor.empty(%4) : tensor<?x8640x3200xf16>
%9 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%8 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %9 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %7 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %5, offsets = [0, 0, 0, 0, 0], sizes = [%4, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%4}
flow.return
} count(%arg2: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2
flow.return %x, %y, %z : index, index, index
}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After OutlineDispatchRegionsPass (iree-flow-outline-dispatch-regions) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0 workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After AnnotateDispatchesPass (iree-flow-annotate-dispatches) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After StripDebugOps (iree-util-strip-debug-ops) //----- //
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
// -----// IR Dump After DeduplicateExecutablesPass (iree-flow-deduplicate-executables) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After InjectTensorTracingPass (iree-flow-inject-tensor-tracing) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After CleanupTensorShapesPass (iree-flow-cleanup-tensor-shapes) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
// -----// IR Dump After CSE (cse) //----- //
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After SymbolDCE (symbol-dce) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyInputPass (iree-stream-verify-input) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After OutlineConstants (iree-util-outline-constants) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
flow.executable private @broadcast_pack_kernel_dispatch_0 {
flow.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
flow.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>) {
%0 = flow.dispatch.workload.ordinal %arg1, 0 : index
%1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
%2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%3 = tensor.empty(%0) : tensor<?x540x3200x16x1xf16>
%4 = tensor.empty(%0) : tensor<?x8640x3200xf16>
%5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %5 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %3 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0, 0], sizes = [%0, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%0}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16>
%2 = flow.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x540x3200x16x1xf16>{%0}
%3 = hal.tensor.export %2 "output0" : tensor<?x540x3200x16x1xf16>{%0} -> !hal.buffer_view
util.return %3 : !hal.buffer_view
}
}
// -----// IR Dump After ConvertToStreamPass (iree-stream-conversion) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
%c8640 = arith.constant 8640 : index
%c3200 = arith.constant 3200 : index
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%c0 = arith.constant 0 : index
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyLoweringToTensorsPass (iree-stream-verify-lowering-to-tensors) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
%c8640 = arith.constant 8640 : index
%c3200 = arith.constant 3200 : index
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%c0 = arith.constant 0 : index
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After CombineInitializers (iree-util-combine-initializers) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.sizeof tensor<8640x3200xf16> : index
%2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1}
%3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1}
%4 = stream.tensor.sizeof tensor<?x540x3200x16x1xf16>{%0} : index
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4}
%6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4}
%7 = stream.tensor.export %6 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view
util.return %7 : !hal.buffer_view
}
}
// -----// IR Dump After EncodeDeviceTensorsPass (iree-stream-encode-device-tensors) //----- //
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
// -----// IR Dump After EncodeHostTensorsPass (iree-stream-encode-host-tensors) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyLoweringToAsyncResourcesPass (iree-stream-verify-lowering-to-async-resources) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
}
// -----// IR Dump After MaterializeCopyOnWritePass (iree-stream-materialize-copy-on-write) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After ElideAsyncCopiesPass (iree-stream-elide-async-copies) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After EmplaceAllocationsPass (iree-stream-emplace-allocations) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000}
%3 = arith.muli %0, %c55296000 : index
%4 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3}
%5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3}
%6 = stream.tensor.export %5 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view
util.return %6 : !hal.buffer_view
}
// -----// IR Dump After RefineUsagePass (iree-stream-refine-usage) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyAsyncAccessRangesPass (iree-stream-verify-async-access-ranges) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After ScheduleExecutionPass (iree-stream-schedule-execution) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After ScheduleConcurrencyPass (iree-stream-schedule-concurrency) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After PropagateTimepointsPass (iree-stream-propagate-timepoints) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.timepoint.immediate => !stream.timepoint
%results, %result_timepoint = stream.async.execute await(%3) => with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%6 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %6 : !stream.resource<external>{%2}
} => !stream.timepoint
%4 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After MaterializeBuiltinsPass (iree-stream-materialize-builtins) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%3 = stream.timepoint.immediate => !stream.timepoint
%results, %result_timepoint = stream.async.execute await(%3) => with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%6 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %6 : !stream.resource<external>{%2}
} => !stream.timepoint
%4 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyLoweringToAsyncPass (iree-stream-verify-lowering-to-async) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} {
%5 = stream.async.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2}
stream.yield %5 : !stream.resource<external>{%2}
} => !stream.timepoint
%3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2}
%4 = stream.tensor.export %3 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %4 : !hal.buffer_view
}
}
// -----// IR Dump After ScheduleAllocationPass (iree-stream-schedule-allocation) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%c0_0 = arith.constant 0 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After PackConstantsPass (iree-stream-pack-constants) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%c0_0 = arith.constant 0 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After LayoutSlicesPass (iree-stream-layout-slices) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%c0_0 = arith.constant 0 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After PropagateSubranges (iree-util-propagate-subranges) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%c0_0 = arith.constant 0 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyLoweringToCmdPass (iree-stream-verify-lowering-to-cmd) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After SCFToControlFlow (convert-scf-to-cf) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After ElideTimepointsPass (iree-stream-elide-timepoints) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FixedPointIterator (iree-util-fixed-point-iterator) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg1, 0 : index
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%0 : index) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After FuseDispatchBindingsPass (iree-stream-fuse-dispatch-bindings) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: index, %arg3: index, %arg4: index) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%arg2] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg4, 0 : index
%2 = stream.binding.subspan %arg1[%arg3] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%c0_0 = arith.constant 0 : index
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0, %c0, %0 : index, index, index) {
ro %arg2[%c0_0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After AnnotateDispatchArgumentsPass (iree-stream-annotate-dispatch-arguments) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: index {stream.values = [0 : index]}, %arg3: index {stream.values = [0 : index]}, %arg4: index) {
%c0 = arith.constant 0 : index
%0 = stream.binding.subspan %arg0[%arg2] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%1 = flow.dispatch.workload.ordinal %arg4, 0 : index
%2 = stream.binding.subspan %arg1[%arg3] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%4 = tensor.empty(%1) : tensor<?x540x3200x16x1xf16>
%5 = tensor.empty(%1) : tensor<?x8640x3200xf16>
%6 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %6 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %4 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %2, offsets = [0, 0, 0, 0, 0], sizes = [%1, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%1}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%c0_0 = arith.constant 0 : index
%3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0, %c0, %0 : index, index, index) {
ro %arg2[%c0_0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2}
%5 = stream.tensor.export %4 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %5 : !hal.buffer_view
}
}
// -----// IR Dump After PackDispatchOperandsPass (iree-stream-pack-dispatch-operands) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) {
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%c32_i64 = arith.constant 32 : i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index
%5 = arith.extui %arg4 : i32 to i64
%6 = arith.extui %arg5 : i32 to i64
%c32_i64_0 = arith.constant 32 : i64
%7 = arith.shli %6, %c32_i64_0 : i64
%8 = arith.ori %5, %7 : i64
%9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index
%10 = arith.extui %arg6 : i32 to i64
%11 = arith.extui %arg7 : i32 to i64
%c32_i64_1 = arith.constant 32 : i64
%12 = arith.shli %11, %c32_i64_1 : i64
%13 = arith.ori %10, %12 : i64
%14 = arith.index_castui %13 : i64 to index
%c0 = arith.constant 0 : index
%15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%16 = flow.dispatch.workload.ordinal %14, 0 : index
%17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
%18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%19 = tensor.empty(%16) : tensor<?x540x3200x16x1xf16>
%20 = tensor.empty(%16) : tensor<?x8640x3200xf16>
%21 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%20 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %21 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %19 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %17, offsets = [0, 0, 0, 0, 0], sizes = [%16, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%c0_0 = arith.constant 0 : index
%c0_i64 = arith.constant 0 : i64
%c0_i32 = arith.constant 0 : i32
%c32_i64 = arith.constant 32 : i64
%c0_i64_1 = arith.constant 0 : i64
%c0_i32_2 = arith.constant 0 : i32
%c0_i64_3 = arith.constant 0 : i64
%c0_i32_4 = arith.constant 0 : i32
%c32_i64_5 = arith.constant 32 : i64
%c0_i64_6 = arith.constant 0 : i64
%c0_i32_7 = arith.constant 0 : i32
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%c32_i64_8 = arith.constant 32 : i64
%5 = arith.shrui %3, %c32_i64_8 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32_2, %c0_i32_4, %c0_i32_7, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0_0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) {
%c32_i64 = arith.constant 32 : i64
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index
%5 = arith.extui %arg4 : i32 to i64
%6 = arith.extui %arg5 : i32 to i64
%7 = arith.shli %6, %c32_i64 : i64
%8 = arith.ori %5, %7 : i64
%9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index
%10 = arith.extui %arg6 : i32 to i64
%11 = arith.extui %arg7 : i32 to i64
%12 = arith.shli %11, %c32_i64 : i64
%13 = arith.ori %10, %12 : i64
%14 = arith.index_castui %13 : i64 to index
%15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%16 = flow.dispatch.workload.ordinal %14, 0 : index
%17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
%18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%19 = tensor.empty(%16) : tensor<?x540x3200x16x1xf16>
%20 = tensor.empty(%16) : tensor<?x8640x3200xf16>
%21 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%20 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %21 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %19 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %17, offsets = [0, 0, 0, 0, 0], sizes = [%16, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) {
%c32_i64 = arith.constant 32 : i64
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index
%5 = arith.extui %arg4 : i32 to i64
%6 = arith.extui %arg5 : i32 to i64
%7 = arith.shli %6, %c32_i64 : i64
%8 = arith.ori %5, %7 : i64
%9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index
%10 = arith.extui %arg6 : i32 to i64
%11 = arith.extui %arg7 : i32 to i64
%12 = arith.shli %11, %c32_i64 : i64
%13 = arith.ori %10, %12 : i64
%14 = arith.index_castui %13 : i64 to index
%15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%16 = flow.dispatch.workload.ordinal %14, 0 : index
%17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
%18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%19 = tensor.empty(%16) : tensor<?x540x3200x16x1xf16>
%20 = tensor.empty(%16) : tensor<?x8640x3200xf16>
%21 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%20 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %21 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %19 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %17, offsets = [0, 0, 0, 0, 0], sizes = [%16, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) {
%c32_i64 = arith.constant 32 : i64
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index
%5 = arith.extui %arg4 : i32 to i64
%6 = arith.extui %arg5 : i32 to i64
%7 = arith.shli %6, %c32_i64 : i64
%8 = arith.ori %5, %7 : i64
%9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index
%10 = arith.extui %arg6 : i32 to i64
%11 = arith.extui %arg7 : i32 to i64
%12 = arith.shli %11, %c32_i64 : i64
%13 = arith.ori %10, %12 : i64
%14 = arith.index_castui %13 : i64 to index
%15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%16 = flow.dispatch.workload.ordinal %14, 0 : index
%17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
%18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%19 = tensor.empty(%16) : tensor<?x540x3200x16x1xf16>
%20 = tensor.empty(%16) : tensor<?x8640x3200xf16>
%21 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%20 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %21 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %19 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %17, offsets = [0, 0, 0, 0, 0], sizes = [%16, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) {
%c32_i64 = arith.constant 32 : i64
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index
%5 = arith.extui %arg4 : i32 to i64
%6 = arith.extui %arg5 : i32 to i64
%7 = arith.shli %6, %c32_i64 : i64
%8 = arith.ori %5, %7 : i64
%9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index
%10 = arith.extui %arg6 : i32 to i64
%11 = arith.extui %arg7 : i32 to i64
%12 = arith.shli %11, %c32_i64 : i64
%13 = arith.ori %10, %12 : i64
%14 = arith.index_castui %13 : i64 to index
%15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%16 = flow.dispatch.workload.ordinal %14, 0 : index
%17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
%18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%19 = tensor.empty(%16) : tensor<?x540x3200x16x1xf16>
%20 = tensor.empty(%16) : tensor<?x8640x3200xf16>
%21 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%20 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %21 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %19 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %17, offsets = [0, 0, 0, 0, 0], sizes = [%16, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FoldUniformOperandsPass (iree-stream-fold-uniform-operands) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c0_i32 = arith.constant 0 : i32
%c32_i64 = arith.constant 32 : i64
%0 = arith.extui %c0_i32 : i32 to i64
%1 = arith.extui %c0_i32 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index
%5 = arith.extui %c0_i32 : i32 to i64
%6 = arith.extui %c0_i32 : i32 to i64
%7 = arith.shli %6, %c32_i64 : i64
%8 = arith.ori %5, %7 : i64
%9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index
%10 = arith.extui %arg2 : i32 to i64
%11 = arith.extui %arg3 : i32 to i64
%12 = arith.shli %11, %c32_i64 : i64
%13 = arith.ori %10, %12 : i64
%14 = arith.index_castui %13 : i64 to index
%15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%16 = flow.dispatch.workload.ordinal %14, 0 : index
%17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
%18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%19 = tensor.empty(%16) : tensor<?x540x3200x16x1xf16>
%20 = tensor.empty(%16) : tensor<?x8640x3200xf16>
%21 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%20 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %21 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %19 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %17, offsets = [0, 0, 0, 0, 0], sizes = [%16, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%16}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c0_i32 = arith.constant 0 : i32
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After CSE (cse) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After IPO (iree-util-ipo) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After SymbolDCE (symbol-dce) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After CSE (cse) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After CSE (cse) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After AssignTargetDevicesPass (iree-hal-assign-target-devices) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After VerifyTargetEnvironmentPass (iree-hal-verify-target-environment) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
stream.executable private @broadcast_pack_kernel_dispatch_0 {
stream.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack workgroups(%arg0: index) -> (index, index, index) {
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0
stream.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%0 = arith.extui %arg2 : i32 to i64
%1 = arith.extui %arg3 : i32 to i64
%2 = arith.shli %1, %c32_i64 : i64
%3 = arith.ori %0, %2 : i64
%4 = arith.index_castui %3 : i64 to index
%5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%6 = flow.dispatch.workload.ordinal %4, 0 : index
%7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%9 = tensor.empty(%6) : tensor<?x540x3200x16x1xf16>
%10 = tensor.empty(%6) : tensor<?x8640x3200xf16>
%11 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%10 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %11 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %9 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %7, offsets = [0, 0, 0, 0, 0], sizes = [%6, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
return
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After MaterializeInterfacesPass (iree-hal-materialize-interfaces) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#pipeline_layout = #hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
hal.executable private @broadcast_pack_kernel_dispatch_0 {
hal.executable.variant public @embedded_elf_x86_64 target(#executable_target_embedded_elf_x86_64_) {
hal.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack ordinal(0) layout(#pipeline_layout) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} {
^bb0(%arg0: !hal.device, %arg1: index):
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1
hal.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@embedded_elf_x86_64::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After PruneExecutablesPass (iree-hal-prune-executables) //----- //
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>
#map = affine_map<(d0, d1, d2) -> (d1, d2)>
#map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
#pipeline_layout = #hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>
#device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]>
module attributes {hal.device.targets = [#device_target_llvm_cpu]} {
hal.executable private @broadcast_pack_kernel_dispatch_0 {
hal.executable.variant public @embedded_elf_x86_64 target(#executable_target_embedded_elf_x86_64_) {
hal.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack ordinal(0) layout(#pipeline_layout) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} {
^bb0(%arg0: !hal.device, %arg1: index):
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1
hal.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
}
}
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@embedded_elf_x86_64::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
}
// -----// IR Dump After CPUMaterializeUpperBoundTileSize (iree-codegen-cpu-materialize-upper-bound-tile-size) //----- //
util.func public @broadcast_pack_kernel(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast_pack_kernel(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x540x3200x16x1xf16>)"}} {
%c32_i64 = arith.constant 32 : i64
%c55296000 = arith.constant 55296000 : index
%c0 = arith.constant 0 : index
%c3200 = arith.constant 3200 : index
%c8640 = arith.constant 8640 : index
%0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index
%element_type_f16 = hal.element_type<f16> : i32
%dense_row_major = hal.encoding_type<dense_row_major> : i32
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major)
%1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000}
%2 = arith.muli %0, %c55296000 : index
%result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint
%3 = arith.index_castui %0 : index to i64
%4 = arith.trunci %3 : i64 to i32
%5 = arith.shrui %3, %c32_i64 : i64
%6 = arith.trunci %5 : i64 to i32
%7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) {
stream.cmd.dispatch @broadcast_pack_kernel_dispatch_0::@embedded_elf_x86_64::@broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack[%0](%4, %6 : i32, i32) {
ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000},
wo %arg3[%c0 for %2] : !stream.resource<external>{%2}
}
} => !stream.timepoint
%8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2}
%9 = stream.tensor.export %8 : tensor<?x540x3200x16x1xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view
util.return %9 : !hal.buffer_view
}
// -----// IR Dump After TypePropagation (iree-codegen-type-propagation) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After BubbleUpOrdinalOps (iree-codegen-bubble-up-ordinal-ops) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After BufferizeCopyOnlyDispatches (iree-codegen-bufferize-copy-only-dispatches) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After DecomposeSoftmax (iree-codegen-decompose-softmax) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After MaterializeUserConfigs (iree-codegen-materialize-user-configs) //----- //
module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
// -----// IR Dump After RematerializeParallelOps (iree-codegen-rematerialize-parallel-ops) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After ExpandF16OpToF32 (iree-llvmcpu-expand-f16-op-to-f32) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After CPUMaterializeEncoding (iree-codegen-cpu-materialize-encoding) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After EraseHALDescriptorTypeFromMemRef (iree-codegen-erase-hal-descriptor-type-from-memref) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
// -----// IR Dump After LLVMCPUSelectLoweringStrategy (iree-llvmcpu-select-lowering-strategy) //----- //
module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
// -----// IR Dump After ConfigureTargetExecutableVariantsPass (iree-hal-configure-target-executable-variants) //----- //
hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) {
hal.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} {
^bb0(%arg0: !hal.device, %arg1: index):
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1
hal.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
}
// -----// IR Dump After ConfigureExecutablesPass (iree-hal-configure-executables) //----- //
hal.executable private @broadcast_pack_kernel_dispatch_0 {
hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) {
hal.executable.export public @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} {
^bb0(%arg0: !hal.device, %arg1: index):
%x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1
hal.return %x, %y, %z : index, index, index
}
builtin.module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
}
}
// -----// IR Dump After LowerExecutableUsingTransformDialect (iree-codegen-lower-executable-using-transform-dialect) //----- //
module {
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = flow.dispatch.workload.ordinal %6, 0 : index
%9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
%10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16>
%11 = tensor.empty(%8) : tensor<?x540x3200x16x1xf16>
%12 = tensor.empty(%8) : tensor<?x8640x3200xf16>
%13 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%12 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x8640x3200xf16>
%pack = tensor.pack %13 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %11 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x8640x3200xf16> -> tensor<?x540x3200x16x1xf16>
flow.dispatch.tensor.store %pack, %9, offsets = [0, 0, 0, 0, 0], sizes = [%8, 540, 3200, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x540x3200x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%8}
return
}
}
// -----// IR Dump After TileAndDistributeToWorkgroups (iree-codegen-tile-and-distribute-to-workgroups) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%17 = flow.dispatch.tensor.load %7, offsets = [%16, %arg2], sizes = [%c64, %c64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<?x?xf16>
%18 = tensor.empty(%11) : tensor<?x64x64xf16>
%cast = tensor.cast %17 : tensor<?x?xf16> to tensor<64x64xf16>
%19 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%cast : tensor<64x64xf16>) outs(%18 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%20 = tensor.empty(%11) : tensor<?x4x64x16x1xf16>
%pack = tensor.pack %19 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %20 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
%cast_0 = tensor.cast %pack : tensor<?x4x64x16x1xf16> to tensor<?x?x?x16x1xf16>
%21 = arith.extui %0 : i32 to i64
%22 = arith.extui %1 : i32 to i64
%23 = arith.shli %22, %c32_i64 : i64
%24 = arith.ori %21, %23 : i64
%25 = arith.index_castui %24 : i64 to index
flow.dispatch.tensor.store %cast_0, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, %c4, %c64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x?x?x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%25}
}
}
}
return
}
// -----// IR Dump After ConvertToDestinationPassingStyle (iree-codegen-convert-to-destination-passing-style) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = arith.extui %0 : i32 to i64
%17 = arith.extui %1 : i32 to i64
%18 = arith.shli %17, %c32_i64 : i64
%19 = arith.ori %16, %18 : i64
%20 = arith.index_castui %19 : i64 to index
%21 = arith.extui %0 : i32 to i64
%22 = arith.extui %1 : i32 to i64
%23 = arith.shli %22, %c32_i64 : i64
%24 = arith.ori %21, %23 : i64
%25 = arith.index_castui %24 : i64 to index
%26 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%25]
%27 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%26, %c4, %c64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%20} -> tensor<?x?x?x16x1xf16>
%cast = tensor.cast %27 : tensor<?x?x?x16x1xf16> to tensor<?x4x64x16x1xf16>
%28 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%29 = flow.dispatch.tensor.load %7, offsets = [%28, %arg2], sizes = [%c64, %c64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<?x?xf16>
%30 = tensor.empty(%11) : tensor<?x64x64xf16>
%cast_0 = tensor.cast %29 : tensor<?x?xf16> to tensor<64x64xf16>
%31 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%cast_0 : tensor<64x64xf16>) outs(%30 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%pack = tensor.pack %31 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %cast {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
%cast_1 = tensor.cast %pack : tensor<?x4x64x16x1xf16> to tensor<?x?x?x16x1xf16>
%32 = arith.extui %0 : i32 to i64
%33 = arith.extui %1 : i32 to i64
%34 = arith.shli %33, %c32_i64 : i64
%35 = arith.ori %32, %34 : i64
%36 = arith.index_castui %35 : i64 to index
flow.dispatch.tensor.store %cast_1, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, %c4, %c64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x?x?x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%36}
}
}
}
return
}
// -----// IR Dump After FoldAffineMinInDistributedLoops (iree-codegen-fold-affinemin-in-distributed-loops) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = arith.extui %0 : i32 to i64
%17 = arith.extui %1 : i32 to i64
%18 = arith.shli %17, %c32_i64 : i64
%19 = arith.ori %16, %18 : i64
%20 = arith.index_castui %19 : i64 to index
%21 = arith.extui %0 : i32 to i64
%22 = arith.extui %1 : i32 to i64
%23 = arith.shli %22, %c32_i64 : i64
%24 = arith.ori %21, %23 : i64
%25 = arith.index_castui %24 : i64 to index
%26 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%25]
%27 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%26, %c4, %c64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%20} -> tensor<?x?x?x16x1xf16>
%cast = tensor.cast %27 : tensor<?x?x?x16x1xf16> to tensor<?x4x64x16x1xf16>
%28 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%29 = flow.dispatch.tensor.load %7, offsets = [%28, %arg2], sizes = [%c64, %c64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<?x?xf16>
%30 = tensor.empty(%11) : tensor<?x64x64xf16>
%cast_0 = tensor.cast %29 : tensor<?x?xf16> to tensor<64x64xf16>
%31 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%cast_0 : tensor<64x64xf16>) outs(%30 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%pack = tensor.pack %31 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %cast {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
%cast_1 = tensor.cast %pack : tensor<?x4x64x16x1xf16> to tensor<?x?x?x16x1xf16>
%32 = arith.extui %0 : i32 to i64
%33 = arith.extui %1 : i32 to i64
%34 = arith.shli %33, %c32_i64 : i64
%35 = arith.ori %32, %34 : i64
%36 = arith.index_castui %35 : i64 to index
flow.dispatch.tensor.store %cast_1, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, %c4, %c64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x?x?x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%36}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = arith.extui %0 : i32 to i64
%17 = arith.extui %1 : i32 to i64
%18 = arith.shli %17, %c32_i64 : i64
%19 = arith.ori %16, %18 : i64
%20 = arith.index_castui %19 : i64 to index
%21 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%20]
%22 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%21, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%23 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%24 = flow.dispatch.tensor.load %7, offsets = [%23, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%25 = tensor.empty(%11) : tensor<?x64x64xf16>
%26 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%24 : tensor<64x64xf16>) outs(%25 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%pack = tensor.pack %26 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %22 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
flow.dispatch.tensor.store %pack, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After CSE (cse) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = tensor.empty(%11) : tensor<?x64x64xf16>
%20 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<64x64xf16>) outs(%19 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%pack = tensor.pack %20 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %16 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
flow.dispatch.tensor.store %pack, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After FuseTensorPadWithConsumer (iree-codegen-fuse-tensor-pad-with-consumer) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = tensor.empty(%11) : tensor<?x64x64xf16>
%20 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<64x64xf16>) outs(%19 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%pack = tensor.pack %20 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %16 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
flow.dispatch.tensor.store %pack, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After ConcretizePadResultShape (iree-codegen-concretize-pad-result-shape) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = tensor.empty(%11) : tensor<?x64x64xf16>
%20 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<64x64xf16>) outs(%19 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<?x64x64xf16>
%pack = tensor.pack %20 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %16 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<?x64x64xf16> -> tensor<?x4x64x16x1xf16>
flow.dispatch.tensor.store %pack, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After LLVMCPUTileAndFuse (iree-llvmcpu-tile-and-fuse) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After FuseTensorPadWithConsumer (iree-codegen-fuse-tensor-pad-with-consumer) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After ConcretizePadResultShape (iree-codegen-concretize-pad-result-shape) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After LLVMCPUSplitReduction (iree-llvmcpu-split-reduction) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After LLVMCPUTile (iree-llvmcpu-tile) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After LLVMCPUTileAndFuse (iree-llvmcpu-tile-and-fuse) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After FuseTensorPadWithConsumer (iree-codegen-fuse-tensor-pad-with-consumer) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After ConcretizePadResultShape (iree-codegen-concretize-pad-result-shape) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After TensorToVectorVectorizePad (iree-codegen-vectorize-tensor-pad) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%pack = tensor.pack %24 outer_dims_perm = [0, 1, 2] inner_dims_pos = [1, 2] inner_tiles = [16, 1] into %extracted_slice_0 {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 4, 64], [1, 1, 16], [0, 0, 0], [0, 0, 0]]>} : tensor<1x16x16xf16> -> tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %pack into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After DecomposePackUnPackOps (iree-codegen-decompose-pack-unpack-ops) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%expanded = tensor.expand_shape %24 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%transposed = linalg.transpose ins(%expanded : tensor<1x1x16x16x1xf16>) outs(%extracted_slice_0 : tensor<1x1x16x16x1xf16>) permutation = [0, 1, 3, 2, 4]
%inserted_slice = tensor.insert_slice %transposed into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%expanded = tensor.expand_shape %24 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%transposed = linalg.transpose ins(%expanded : tensor<1x1x16x16x1xf16>) outs(%extracted_slice_0 : tensor<1x1x16x16x1xf16>) permutation = [0, 1, 3, 2, 4]
%inserted_slice = tensor.insert_slice %transposed into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After CSE (cse) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%23 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
} -> tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%expanded = tensor.expand_shape %24 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%transposed = linalg.transpose ins(%expanded : tensor<1x1x16x16x1xf16>) outs(%extracted_slice_0 : tensor<1x1x16x16x1xf16>) permutation = [0, 1, 3, 2, 4]
%inserted_slice = tensor.insert_slice %transposed into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After GenericVectorization (iree-codegen-generic-vectorization) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant 0.000000e+00 : f16
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
scf.for %arg0 = %9 to %6 step %10 {
%11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
scf.for %arg1 = %12 to %c540 step %13 {
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg2 = %14 to %c3200 step %15 {
%16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
%18 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%19 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x4x64x16x1xf16>) {
%20 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%extracted_slice = tensor.extract_slice %18[%22, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16>
%23 = tensor.empty() : tensor<1x16x16xf16>
%24 = vector.transfer_read %extracted_slice[%c0, %c0], %cst {in_bounds = [true, true]} : tensor<16x16xf16>, vector<16x16xf16>
%25 = vector.broadcast %24 : vector<16x16xf16> to vector<1x16x16xf16>
%26 = vector.transfer_write %25, %23[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16>
%extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> to tensor<1x1x16x16x1xf16>
%expanded = tensor.expand_shape %26 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%27 = vector.transfer_read %expanded[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : tensor<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%28 = vector.transpose %27, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%29 = vector.transfer_write %28, %extracted_slice_0[%c0, %c0, %c0, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, tensor<1x1x16x16x1xf16>
%inserted_slice = tensor.insert_slice %29 into %arg8[%arg3, %arg5, %arg7, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : tensor<1x1x16x16x1xf16> into tensor<?x4x64x16x1xf16>
scf.yield %inserted_slice : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
scf.yield %20 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %19, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%11, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After OptimizeTensorInsertExtractSlices (iree-codegen-optimize-tensor-insert-extract-slices) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant 0.000000e+00 : f16
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
%15 = tensor.empty() : tensor<1x16x16xf16>
scf.for %arg0 = %9 to %6 step %10 {
%16 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%18 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%19 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%20 = scf.for %arg3 = %c0 to %16 step %c1 iter_args(%arg4 = %18) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%23 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%24 = vector.transfer_read %19[%22, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16>
%25 = vector.broadcast %24 : vector<16x16xf16> to vector<1x16x16xf16>
%26 = vector.transfer_write %25, %15[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16>
%expanded = tensor.expand_shape %26 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%27 = vector.transfer_read %expanded[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : tensor<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%28 = vector.transpose %27, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%29 = vector.transfer_write %28, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, tensor<?x4x64x16x1xf16>
scf.yield %29 : tensor<?x4x64x16x1xf16>
}
scf.yield %23 : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %20, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant 0.000000e+00 : f16
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
%15 = tensor.empty() : tensor<1x16x16xf16>
scf.for %arg0 = %9 to %6 step %10 {
%16 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%18 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%19 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%20 = scf.for %arg3 = %c0 to %16 step %c1 iter_args(%arg4 = %18) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%23 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%24 = vector.transfer_read %19[%22, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16>
%25 = vector.broadcast %24 : vector<16x16xf16> to vector<1x16x16xf16>
%26 = vector.transfer_write %25, %15[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16>
%expanded = tensor.expand_shape %26 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%27 = vector.transfer_read %expanded[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : tensor<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%28 = vector.transpose %27, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%29 = vector.transfer_write %28, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, tensor<?x4x64x16x1xf16>
scf.yield %29 : tensor<?x4x64x16x1xf16>
}
scf.yield %23 : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %20, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After CSE (cse) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant 0.000000e+00 : f16
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
%15 = tensor.empty() : tensor<1x16x16xf16>
scf.for %arg0 = %9 to %6 step %10 {
%16 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%18 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%19 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%20 = scf.for %arg3 = %c0 to %16 step %c1 iter_args(%arg4 = %18) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%23 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%24 = vector.transfer_read %19[%22, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16>
%25 = vector.broadcast %24 : vector<16x16xf16> to vector<1x16x16xf16>
%26 = vector.transfer_write %25, %15[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16>
%expanded = tensor.expand_shape %26 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%27 = vector.transfer_read %expanded[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : tensor<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%28 = vector.transpose %27, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%29 = vector.transfer_write %28, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, tensor<?x4x64x16x1xf16>
scf.yield %29 : tensor<?x4x64x16x1xf16>
}
scf.yield %23 : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %20, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After EliminateEmptyTensors (iree-eliminate-empty-tensors) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant 0.000000e+00 : f16
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
%15 = tensor.empty() : tensor<1x16x16xf16>
scf.for %arg0 = %9 to %6 step %10 {
%16 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%18 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%19 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%20 = scf.for %arg3 = %c0 to %16 step %c1 iter_args(%arg4 = %18) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%23 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%24 = vector.transfer_read %19[%22, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16>
%25 = vector.broadcast %24 : vector<16x16xf16> to vector<1x16x16xf16>
%26 = vector.transfer_write %25, %15[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16>
%expanded = tensor.expand_shape %26 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%27 = vector.transfer_read %expanded[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : tensor<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%28 = vector.transpose %27, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%29 = vector.transfer_write %28, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, tensor<?x4x64x16x1xf16>
scf.yield %29 : tensor<?x4x64x16x1xf16>
}
scf.yield %23 : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %20, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After EmptyTensorToAllocTensor (empty-tensor-to-alloc-tensor) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant 0.000000e+00 : f16
%c16 = arith.constant 16 : index
%c64 = arith.constant 64 : index
%c4 = arith.constant 4 : index
%c1 = arith.constant 1 : index
%c3200 = arith.constant 3200 : index
%c540 = arith.constant 540 : index
%c0 = arith.constant 0 : index
%c32_i64 = arith.constant 32 : i64
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
%15 = bufferization.alloc_tensor() : tensor<1x16x16xf16>
scf.for %arg0 = %9 to %6 step %10 {
%16 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%18 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6} -> tensor<?x4x64x16x1xf16>
%19 = flow.dispatch.tensor.load %7, offsets = [%17, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16>
%20 = scf.for %arg3 = %c0 to %16 step %c1 iter_args(%arg4 = %18) -> (tensor<?x4x64x16x1xf16>) {
%21 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (tensor<?x4x64x16x1xf16>) {
%22 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%23 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x4x64x16x1xf16>) {
%24 = vector.transfer_read %19[%22, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16>
%25 = vector.broadcast %24 : vector<16x16xf16> to vector<1x16x16xf16>
%26 = vector.transfer_write %25, %15[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16>
%expanded = tensor.expand_shape %26 [[0], [1, 2], [3, 4]] : tensor<1x16x16xf16> into tensor<1x1x16x16x1xf16>
%27 = vector.transfer_read %expanded[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : tensor<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%28 = vector.transpose %27, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%29 = vector.transfer_write %28, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, tensor<?x4x64x16x1xf16>
scf.yield %29 : tensor<?x4x64x16x1xf16>
}
scf.yield %23 : tensor<?x4x64x16x1xf16>
}
scf.yield %21 : tensor<?x4x64x16x1xf16>
}
flow.dispatch.tensor.store %20, %8, offsets = [%arg0, %arg1, %arg2, 0, 0], sizes = [%16, 4, 64, 16, 1], strides = [1, 1, 1, 1, 1] : tensor<?x4x64x16x1xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x540x3200x16x1xf16>>{%6}
}
}
}
return
}
// -----// IR Dump After IREEComprehensiveBufferize (iree-codegen-iree-comprehensive-bufferize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%17 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %subview) -> (memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
%18 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
%19 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
%21 = vector.transfer_read %subview_0[%19, %arg7], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %22, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%23 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%24 = vector.transpose %23, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %24, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.yield %arg8 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
scf.yield %20 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
scf.yield %18 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
%subview_1 = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%17 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview_1 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
}
}
}
}
return
}
// -----// IR Dump After ResolveShapedTypeResultDims (resolve-shaped-type-result-dims) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%17 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %subview) -> (memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
%18 = scf.for %arg5 = %c0 to %c4 step %c1 iter_args(%arg6 = %arg4) -> (memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
%19 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg5)
%20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
%21 = vector.transfer_read %subview_0[%19, %arg7], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %22, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%23 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%24 = vector.transpose %23, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %24, %arg8[%arg3, %arg5, %arg7, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.yield %arg8 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
scf.yield %20 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
scf.yield %18 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
%subview_1 = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%17 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview_1 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%19 = vector.broadcast %18 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %19, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%20 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%21 = vector.transpose %20, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %21, %subview[%arg3, %arg4, %arg5, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
%subview_1 = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%subview : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview_1 : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
}
}
}
}
return
}
// -----// IR Dump After CSE (cse) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%19 = vector.broadcast %18 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %19, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%20 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%21 = vector.transpose %20, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %21, %subview[%arg3, %arg4, %arg5, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
linalg.generic {indexing_maps = [affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>, affine_map<(d0, d1, d2, d3, d4) -> (d0, d1, d2, d3, d4)>], iterator_types = ["parallel", "parallel", "parallel", "parallel", "parallel"]} ins(%subview : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) {
^bb0(%in: f16, %out: f16):
linalg.yield %in : f16
}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%19 = vector.broadcast %18 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %19, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%20 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%21 = vector.transpose %20, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %21, %subview[%arg3, %arg4, %arg5, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
}
}
}
return
}
// -----// IR Dump After CleanupBufferAllocView (iree-codegen-cleanup-buffer-alloc-view) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%19 = vector.broadcast %18 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %19, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%20 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%21 = vector.transpose %20, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %21, %subview[%arg3, %arg4, %arg5, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
}
}
}
return
}
// -----// IR Dump After RemoveSingleIterationLoop (iree-codegen-remove-single-iteration-loop) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%19 = vector.broadcast %18 : vector<16x16xf16> to vector<1x16x16xf16>
vector.transfer_write %19, %alloca[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<1x16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%20 = vector.transfer_read %expand_shape[%c0, %c0, %c0, %c0, %c0], %cst {in_bounds = [true, true, true, true, true]} : memref<1x1x16x16x1xf16>, vector<1x1x16x16x1xf16>
%21 = vector.transpose %20, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
vector.transfer_write %21, %subview[%arg3, %arg4, %arg5, %c0, %c0] {in_bounds = [true, true, true, true, true]} : vector<1x1x16x16x1xf16>, memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
}
}
}
return
}
// -----// IR Dump After LLVMCPUDropVectorUnitDims (iree-llvmcpu-drop-vector-unit-dims) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%subview_1 = memref.subview %alloca[0, 0, 0] [1, 16, 16] [1, 1, 1] : memref<1x16x16xf16> to memref<16x16xf16>
vector.transfer_write %18, %subview_1[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf16>, memref<16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%subview_2 = memref.subview %expand_shape[0, 0, 0, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : memref<1x1x16x16x1xf16> to memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>>
%subview_3 = memref.subview %subview_2[0, 0, 0, 0] [1, 1, 16, 16] [1, 1, 1, 1] : memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>> to memref<16x16xf16>
%19 = vector.transfer_read %subview_3[%c0, %c0], %cst {in_bounds = [true, true]} : memref<16x16xf16>, vector<16x16xf16>
%20 = vector.shape_cast %19 : vector<16x16xf16> to vector<16x16x1xf16>
%21 = vector.broadcast %20 : vector<16x16x1xf16> to vector<1x1x16x16x1xf16>
%22 = vector.transpose %21, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%23 = vector.extract %22[0, 0] : vector<16x16x1xf16> from vector<1x1x16x16x1xf16>
%subview_4 = memref.subview %subview[0, 0, 0, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%24 = vector.shape_cast %23 : vector<16x16x1xf16> to vector<16x16xf16>
vector.transfer_write %24, %subview_4[%arg3, %arg4, %arg5, %c0] {in_bounds = [true, true]} : vector<16x16xf16>, memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
}
}
}
return
}
// -----// IR Dump After LLVMCPUVirtualVectorLowering (iree-llvmcpu-virtual-vector-lowering) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%subview_1 = memref.subview %alloca[0, 0, 0] [1, 16, 16] [1, 1, 1] : memref<1x16x16xf16> to memref<16x16xf16>
vector.transfer_write %18, %subview_1[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf16>, memref<16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%subview_2 = memref.subview %expand_shape[0, 0, 0, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : memref<1x1x16x16x1xf16> to memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>>
%subview_3 = memref.subview %subview_2[0, 0, 0, 0] [1, 1, 16, 16] [1, 1, 1, 1] : memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>> to memref<16x16xf16>
%19 = vector.transfer_read %subview_3[%c0, %c0], %cst {in_bounds = [true, true]} : memref<16x16xf16>, vector<16x16xf16>
%20 = vector.shape_cast %19 : vector<16x16xf16> to vector<16x16x1xf16>
%21 = vector.broadcast %20 : vector<16x16x1xf16> to vector<1x1x16x16x1xf16>
%22 = vector.transpose %21, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%23 = vector.extract %22[0, 0] : vector<16x16x1xf16> from vector<1x1x16x16x1xf16>
%subview_4 = memref.subview %subview[0, 0, 0, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%24 = vector.shape_cast %23 : vector<16x16x1xf16> to vector<16x16xf16>
vector.transfer_write %24, %subview_4[%arg3, %arg4, %arg5, %c0] {in_bounds = [true, true]} : vector<16x16xf16>, memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%cst = arith.constant 0.000000e+00 : f16
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
scf.for %arg5 = %c0 to %c64 step %c16 {
%18 = vector.transfer_read %subview_0[%17, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16>
%subview_1 = memref.subview %alloca[0, 0, 0] [1, 16, 16] [1, 1, 1] : memref<1x16x16xf16> to memref<16x16xf16>
vector.transfer_write %18, %subview_1[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf16>, memref<16x16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%subview_2 = memref.subview %expand_shape[0, 0, 0, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : memref<1x1x16x16x1xf16> to memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>>
%subview_3 = memref.subview %subview_2[0, 0, 0, 0] [1, 1, 16, 16] [1, 1, 1, 1] : memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>> to memref<16x16xf16>
%19 = vector.transfer_read %subview_3[%c0, %c0], %cst {in_bounds = [true, true]} : memref<16x16xf16>, vector<16x16xf16>
%20 = vector.shape_cast %19 : vector<16x16xf16> to vector<16x16x1xf16>
%21 = vector.broadcast %20 : vector<16x16x1xf16> to vector<1x1x16x16x1xf16>
%22 = vector.transpose %21, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%23 = vector.extract %22[0, 0] : vector<16x16x1xf16> from vector<1x1x16x16x1xf16>
%subview_4 = memref.subview %subview[0, 0, 0, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%24 = vector.shape_cast %23 : vector<16x16x1xf16> to vector<16x16xf16>
vector.transfer_write %24, %subview_4[%arg3, %arg4, %arg5, %c0] {in_bounds = [true, true]} : vector<16x16xf16>, memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
}
}
}
}
}
}
return
}
// -----// IR Dump After LLVMCPUVectorTransferLowering (iree-llvmcpu-vector-transfer-lowering) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant dense<0.000000e+00> : vector<16x16xf16>
%c15 = arith.constant 15 : index
%c14 = arith.constant 14 : index
%c13 = arith.constant 13 : index
%c12 = arith.constant 12 : index
%c11 = arith.constant 11 : index
%c10 = arith.constant 10 : index
%c9 = arith.constant 9 : index
%c8 = arith.constant 8 : index
%c7 = arith.constant 7 : index
%c6 = arith.constant 6 : index
%c5 = arith.constant 5 : index
%c3 = arith.constant 3 : index
%c2 = arith.constant 2 : index
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
scf.for %arg5 = %c0 to %c64 step %c16 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
%18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%19 = affine.apply affine_map<(d0) -> (d0 * 16 + 1)>(%arg4)
%20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%21 = affine.apply affine_map<(d0) -> (d0 * 16 + 2)>(%arg4)
%22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%23 = affine.apply affine_map<(d0) -> (d0 * 16 + 3)>(%arg4)
%24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%25 = affine.apply affine_map<(d0) -> (d0 * 16 + 4)>(%arg4)
%26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%27 = affine.apply affine_map<(d0) -> (d0 * 16 + 5)>(%arg4)
%28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%29 = affine.apply affine_map<(d0) -> (d0 * 16 + 6)>(%arg4)
%30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%31 = affine.apply affine_map<(d0) -> (d0 * 16 + 7)>(%arg4)
%32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%33 = affine.apply affine_map<(d0) -> (d0 * 16 + 8)>(%arg4)
%34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%35 = affine.apply affine_map<(d0) -> (d0 * 16 + 9)>(%arg4)
%36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%37 = affine.apply affine_map<(d0) -> (d0 * 16 + 10)>(%arg4)
%38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%39 = affine.apply affine_map<(d0) -> (d0 * 16 + 11)>(%arg4)
%40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%41 = affine.apply affine_map<(d0) -> (d0 * 16 + 12)>(%arg4)
%42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%43 = affine.apply affine_map<(d0) -> (d0 * 16 + 13)>(%arg4)
%44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%45 = affine.apply affine_map<(d0) -> (d0 * 16 + 14)>(%arg4)
%46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%47 = affine.apply affine_map<(d0) -> (d0 * 16 + 15)>(%arg4)
%48 = vector.load %subview_0[%47, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%subview_1 = memref.subview %alloca[0, 0, 0] [1, 16, 16] [1, 1, 1] : memref<1x16x16xf16> to memref<16x16xf16>
vector.store %18, %subview_1[%c0, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %20, %subview_1[%c1, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %22, %subview_1[%c2, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %24, %subview_1[%c3, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %26, %subview_1[%c4, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %28, %subview_1[%c5, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %30, %subview_1[%c6, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %32, %subview_1[%c7, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %34, %subview_1[%c8, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %36, %subview_1[%c9, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %38, %subview_1[%c10, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %40, %subview_1[%c11, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %42, %subview_1[%c12, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %44, %subview_1[%c13, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %46, %subview_1[%c14, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %48, %subview_1[%c15, %c0] : memref<16x16xf16>, vector<16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%subview_2 = memref.subview %expand_shape[0, 0, 0, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : memref<1x1x16x16x1xf16> to memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>>
%subview_3 = memref.subview %subview_2[0, 0, 0, 0] [1, 1, 16, 16] [1, 1, 1, 1] : memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>> to memref<16x16xf16>
%49 = vector.load %subview_3[%c0, %c0] : memref<16x16xf16>, vector<16xf16>
%50 = vector.insert %49, %cst [0] : vector<16xf16> into vector<16x16xf16>
%51 = vector.load %subview_3[%c1, %c0] : memref<16x16xf16>, vector<16xf16>
%52 = vector.insert %51, %50 [1] : vector<16xf16> into vector<16x16xf16>
%53 = vector.load %subview_3[%c2, %c0] : memref<16x16xf16>, vector<16xf16>
%54 = vector.insert %53, %52 [2] : vector<16xf16> into vector<16x16xf16>
%55 = vector.load %subview_3[%c3, %c0] : memref<16x16xf16>, vector<16xf16>
%56 = vector.insert %55, %54 [3] : vector<16xf16> into vector<16x16xf16>
%57 = vector.load %subview_3[%c4, %c0] : memref<16x16xf16>, vector<16xf16>
%58 = vector.insert %57, %56 [4] : vector<16xf16> into vector<16x16xf16>
%59 = vector.load %subview_3[%c5, %c0] : memref<16x16xf16>, vector<16xf16>
%60 = vector.insert %59, %58 [5] : vector<16xf16> into vector<16x16xf16>
%61 = vector.load %subview_3[%c6, %c0] : memref<16x16xf16>, vector<16xf16>
%62 = vector.insert %61, %60 [6] : vector<16xf16> into vector<16x16xf16>
%63 = vector.load %subview_3[%c7, %c0] : memref<16x16xf16>, vector<16xf16>
%64 = vector.insert %63, %62 [7] : vector<16xf16> into vector<16x16xf16>
%65 = vector.load %subview_3[%c8, %c0] : memref<16x16xf16>, vector<16xf16>
%66 = vector.insert %65, %64 [8] : vector<16xf16> into vector<16x16xf16>
%67 = vector.load %subview_3[%c9, %c0] : memref<16x16xf16>, vector<16xf16>
%68 = vector.insert %67, %66 [9] : vector<16xf16> into vector<16x16xf16>
%69 = vector.load %subview_3[%c10, %c0] : memref<16x16xf16>, vector<16xf16>
%70 = vector.insert %69, %68 [10] : vector<16xf16> into vector<16x16xf16>
%71 = vector.load %subview_3[%c11, %c0] : memref<16x16xf16>, vector<16xf16>
%72 = vector.insert %71, %70 [11] : vector<16xf16> into vector<16x16xf16>
%73 = vector.load %subview_3[%c12, %c0] : memref<16x16xf16>, vector<16xf16>
%74 = vector.insert %73, %72 [12] : vector<16xf16> into vector<16x16xf16>
%75 = vector.load %subview_3[%c13, %c0] : memref<16x16xf16>, vector<16xf16>
%76 = vector.insert %75, %74 [13] : vector<16xf16> into vector<16x16xf16>
%77 = vector.load %subview_3[%c14, %c0] : memref<16x16xf16>, vector<16xf16>
%78 = vector.insert %77, %76 [14] : vector<16xf16> into vector<16x16xf16>
%79 = vector.load %subview_3[%c15, %c0] : memref<16x16xf16>, vector<16xf16>
%80 = vector.insert %79, %78 [15] : vector<16xf16> into vector<16x16xf16>
%81 = vector.shape_cast %80 : vector<16x16xf16> to vector<16x16x1xf16>
%82 = vector.broadcast %81 : vector<16x16x1xf16> to vector<1x1x16x16x1xf16>
%83 = vector.transpose %82, [0, 1, 3, 2, 4] : vector<1x1x16x16x1xf16> to vector<1x1x16x16x1xf16>
%84 = vector.extract %83[0, 0] : vector<16x16x1xf16> from vector<1x1x16x16x1xf16>
%subview_4 = memref.subview %subview[0, 0, 0, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%85 = vector.shape_cast %84 : vector<16x16x1xf16> to vector<16x16xf16>
%86 = vector.extract %85[0] : vector<16xf16> from vector<16x16xf16>
vector.store %86, %subview_4[%arg3, %arg4, %arg5, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%87 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg5)
%88 = vector.extract %85[1] : vector<16xf16> from vector<16x16xf16>
vector.store %88, %subview_4[%arg3, %arg4, %87, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%89 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg5)
%90 = vector.extract %85[2] : vector<16xf16> from vector<16x16xf16>
vector.store %90, %subview_4[%arg3, %arg4, %89, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%91 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg5)
%92 = vector.extract %85[3] : vector<16xf16> from vector<16x16xf16>
vector.store %92, %subview_4[%arg3, %arg4, %91, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%93 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg5)
%94 = vector.extract %85[4] : vector<16xf16> from vector<16x16xf16>
vector.store %94, %subview_4[%arg3, %arg4, %93, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%95 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg5)
%96 = vector.extract %85[5] : vector<16xf16> from vector<16x16xf16>
vector.store %96, %subview_4[%arg3, %arg4, %95, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%97 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg5)
%98 = vector.extract %85[6] : vector<16xf16> from vector<16x16xf16>
vector.store %98, %subview_4[%arg3, %arg4, %97, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%99 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg5)
%100 = vector.extract %85[7] : vector<16xf16> from vector<16x16xf16>
vector.store %100, %subview_4[%arg3, %arg4, %99, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%101 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg5)
%102 = vector.extract %85[8] : vector<16xf16> from vector<16x16xf16>
vector.store %102, %subview_4[%arg3, %arg4, %101, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%103 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg5)
%104 = vector.extract %85[9] : vector<16xf16> from vector<16x16xf16>
vector.store %104, %subview_4[%arg3, %arg4, %103, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%105 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg5)
%106 = vector.extract %85[10] : vector<16xf16> from vector<16x16xf16>
vector.store %106, %subview_4[%arg3, %arg4, %105, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%107 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg5)
%108 = vector.extract %85[11] : vector<16xf16> from vector<16x16xf16>
vector.store %108, %subview_4[%arg3, %arg4, %107, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%109 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg5)
%110 = vector.extract %85[12] : vector<16xf16> from vector<16x16xf16>
vector.store %110, %subview_4[%arg3, %arg4, %109, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%111 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg5)
%112 = vector.extract %85[13] : vector<16xf16> from vector<16x16xf16>
vector.store %112, %subview_4[%arg3, %arg4, %111, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%113 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg5)
%114 = vector.extract %85[14] : vector<16xf16> from vector<16x16xf16>
vector.store %114, %subview_4[%arg3, %arg4, %113, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%115 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg5)
%116 = vector.extract %85[15] : vector<16xf16> from vector<16x16xf16>
vector.store %116, %subview_4[%arg3, %arg4, %115, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
}
}
}
}
}
}
return
}
// -----// IR Dump After LLVMCPUVectorTransposeLowering (iree-llvmcpu-vector-transpose-lowering) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%cst = arith.constant dense<0.000000e+00> : vector<16x16xf16>
%c15 = arith.constant 15 : index
%c14 = arith.constant 14 : index
%c13 = arith.constant 13 : index
%c12 = arith.constant 12 : index
%c11 = arith.constant 11 : index
%c10 = arith.constant 10 : index
%c9 = arith.constant 9 : index
%c8 = arith.constant 8 : index
%c7 = arith.constant 7 : index
%c6 = arith.constant 6 : index
%c5 = arith.constant 5 : index
%c3 = arith.constant 3 : index
%c2 = arith.constant 2 : index
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
scf.for %arg5 = %c0 to %c64 step %c16 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
%18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%19 = affine.apply affine_map<(d0) -> (d0 * 16 + 1)>(%arg4)
%20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%21 = affine.apply affine_map<(d0) -> (d0 * 16 + 2)>(%arg4)
%22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%23 = affine.apply affine_map<(d0) -> (d0 * 16 + 3)>(%arg4)
%24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%25 = affine.apply affine_map<(d0) -> (d0 * 16 + 4)>(%arg4)
%26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%27 = affine.apply affine_map<(d0) -> (d0 * 16 + 5)>(%arg4)
%28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%29 = affine.apply affine_map<(d0) -> (d0 * 16 + 6)>(%arg4)
%30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%31 = affine.apply affine_map<(d0) -> (d0 * 16 + 7)>(%arg4)
%32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%33 = affine.apply affine_map<(d0) -> (d0 * 16 + 8)>(%arg4)
%34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%35 = affine.apply affine_map<(d0) -> (d0 * 16 + 9)>(%arg4)
%36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%37 = affine.apply affine_map<(d0) -> (d0 * 16 + 10)>(%arg4)
%38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%39 = affine.apply affine_map<(d0) -> (d0 * 16 + 11)>(%arg4)
%40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%41 = affine.apply affine_map<(d0) -> (d0 * 16 + 12)>(%arg4)
%42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%43 = affine.apply affine_map<(d0) -> (d0 * 16 + 13)>(%arg4)
%44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%45 = affine.apply affine_map<(d0) -> (d0 * 16 + 14)>(%arg4)
%46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%47 = affine.apply affine_map<(d0) -> (d0 * 16 + 15)>(%arg4)
%48 = vector.load %subview_0[%47, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%subview_1 = memref.subview %alloca[0, 0, 0] [1, 16, 16] [1, 1, 1] : memref<1x16x16xf16> to memref<16x16xf16>
vector.store %18, %subview_1[%c0, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %20, %subview_1[%c1, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %22, %subview_1[%c2, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %24, %subview_1[%c3, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %26, %subview_1[%c4, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %28, %subview_1[%c5, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %30, %subview_1[%c6, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %32, %subview_1[%c7, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %34, %subview_1[%c8, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %36, %subview_1[%c9, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %38, %subview_1[%c10, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %40, %subview_1[%c11, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %42, %subview_1[%c12, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %44, %subview_1[%c13, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %46, %subview_1[%c14, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %48, %subview_1[%c15, %c0] : memref<16x16xf16>, vector<16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%subview_2 = memref.subview %expand_shape[0, 0, 0, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : memref<1x1x16x16x1xf16> to memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>>
%subview_3 = memref.subview %subview_2[0, 0, 0, 0] [1, 1, 16, 16] [1, 1, 1, 1] : memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>> to memref<16x16xf16>
%49 = vector.load %subview_3[%c0, %c0] : memref<16x16xf16>, vector<16xf16>
%50 = vector.insert %49, %cst [0] : vector<16xf16> into vector<16x16xf16>
%51 = vector.load %subview_3[%c1, %c0] : memref<16x16xf16>, vector<16xf16>
%52 = vector.insert %51, %50 [1] : vector<16xf16> into vector<16x16xf16>
%53 = vector.load %subview_3[%c2, %c0] : memref<16x16xf16>, vector<16xf16>
%54 = vector.insert %53, %52 [2] : vector<16xf16> into vector<16x16xf16>
%55 = vector.load %subview_3[%c3, %c0] : memref<16x16xf16>, vector<16xf16>
%56 = vector.insert %55, %54 [3] : vector<16xf16> into vector<16x16xf16>
%57 = vector.load %subview_3[%c4, %c0] : memref<16x16xf16>, vector<16xf16>
%58 = vector.insert %57, %56 [4] : vector<16xf16> into vector<16x16xf16>
%59 = vector.load %subview_3[%c5, %c0] : memref<16x16xf16>, vector<16xf16>
%60 = vector.insert %59, %58 [5] : vector<16xf16> into vector<16x16xf16>
%61 = vector.load %subview_3[%c6, %c0] : memref<16x16xf16>, vector<16xf16>
%62 = vector.insert %61, %60 [6] : vector<16xf16> into vector<16x16xf16>
%63 = vector.load %subview_3[%c7, %c0] : memref<16x16xf16>, vector<16xf16>
%64 = vector.insert %63, %62 [7] : vector<16xf16> into vector<16x16xf16>
%65 = vector.load %subview_3[%c8, %c0] : memref<16x16xf16>, vector<16xf16>
%66 = vector.insert %65, %64 [8] : vector<16xf16> into vector<16x16xf16>
%67 = vector.load %subview_3[%c9, %c0] : memref<16x16xf16>, vector<16xf16>
%68 = vector.insert %67, %66 [9] : vector<16xf16> into vector<16x16xf16>
%69 = vector.load %subview_3[%c10, %c0] : memref<16x16xf16>, vector<16xf16>
%70 = vector.insert %69, %68 [10] : vector<16xf16> into vector<16x16xf16>
%71 = vector.load %subview_3[%c11, %c0] : memref<16x16xf16>, vector<16xf16>
%72 = vector.insert %71, %70 [11] : vector<16xf16> into vector<16x16xf16>
%73 = vector.load %subview_3[%c12, %c0] : memref<16x16xf16>, vector<16xf16>
%74 = vector.insert %73, %72 [12] : vector<16xf16> into vector<16x16xf16>
%75 = vector.load %subview_3[%c13, %c0] : memref<16x16xf16>, vector<16xf16>
%76 = vector.insert %75, %74 [13] : vector<16xf16> into vector<16x16xf16>
%77 = vector.load %subview_3[%c14, %c0] : memref<16x16xf16>, vector<16xf16>
%78 = vector.insert %77, %76 [14] : vector<16xf16> into vector<16x16xf16>
%79 = vector.load %subview_3[%c15, %c0] : memref<16x16xf16>, vector<16xf16>
%80 = vector.insert %79, %78 [15] : vector<16xf16> into vector<16x16xf16>
%81 = vector.shape_cast %80 : vector<16x16xf16> to vector<16x16x1xf16>
%82 = vector.broadcast %81 : vector<16x16x1xf16> to vector<1x1x16x16x1xf16>
%83 = vector.shape_cast %82 : vector<1x1x16x16x1xf16> to vector<16x16xf16>
%84 = vector.extract %83[0] : vector<16xf16> from vector<16x16xf16>
%85 = vector.extract %83[1] : vector<16xf16> from vector<16x16xf16>
%86 = vector.extract %83[2] : vector<16xf16> from vector<16x16xf16>
%87 = vector.extract %83[3] : vector<16xf16> from vector<16x16xf16>
%88 = vector.extract %83[4] : vector<16xf16> from vector<16x16xf16>
%89 = vector.extract %83[5] : vector<16xf16> from vector<16x16xf16>
%90 = vector.extract %83[6] : vector<16xf16> from vector<16x16xf16>
%91 = vector.extract %83[7] : vector<16xf16> from vector<16x16xf16>
%92 = vector.extract %83[8] : vector<16xf16> from vector<16x16xf16>
%93 = vector.extract %83[9] : vector<16xf16> from vector<16x16xf16>
%94 = vector.extract %83[10] : vector<16xf16> from vector<16x16xf16>
%95 = vector.extract %83[11] : vector<16xf16> from vector<16x16xf16>
%96 = vector.extract %83[12] : vector<16xf16> from vector<16x16xf16>
%97 = vector.extract %83[13] : vector<16xf16> from vector<16x16xf16>
%98 = vector.extract %83[14] : vector<16xf16> from vector<16x16xf16>
%99 = vector.extract %83[15] : vector<16xf16> from vector<16x16xf16>
%100 = vector.shuffle %84, %85 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%101 = vector.shuffle %84, %85 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%102 = vector.shuffle %86, %87 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%103 = vector.shuffle %86, %87 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%104 = vector.shuffle %88, %89 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%105 = vector.shuffle %88, %89 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%106 = vector.shuffle %90, %91 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%107 = vector.shuffle %90, %91 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%108 = vector.shuffle %92, %93 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%109 = vector.shuffle %92, %93 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%110 = vector.shuffle %94, %95 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%111 = vector.shuffle %94, %95 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%112 = vector.shuffle %96, %97 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%113 = vector.shuffle %96, %97 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%114 = vector.shuffle %98, %99 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%115 = vector.shuffle %98, %99 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%116 = vector.shuffle %100, %102 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%117 = vector.shuffle %100, %102 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%118 = vector.shuffle %101, %103 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%119 = vector.shuffle %101, %103 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%120 = vector.shuffle %104, %106 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%121 = vector.shuffle %104, %106 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%122 = vector.shuffle %105, %107 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%123 = vector.shuffle %105, %107 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%124 = vector.shuffle %108, %110 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%125 = vector.shuffle %108, %110 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%126 = vector.shuffle %109, %111 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%127 = vector.shuffle %109, %111 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%128 = vector.shuffle %112, %114 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%129 = vector.shuffle %112, %114 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%130 = vector.shuffle %113, %115 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%131 = vector.shuffle %113, %115 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%132 = vector.shuffle %116, %120 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%133 = vector.shuffle %117, %121 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%134 = vector.shuffle %118, %122 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%135 = vector.shuffle %119, %123 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%136 = vector.shuffle %116, %120 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%137 = vector.shuffle %117, %121 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%138 = vector.shuffle %118, %122 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%139 = vector.shuffle %119, %123 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%140 = vector.shuffle %124, %128 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%141 = vector.shuffle %125, %129 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%142 = vector.shuffle %126, %130 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%143 = vector.shuffle %127, %131 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%144 = vector.shuffle %124, %128 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%145 = vector.shuffle %125, %129 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%146 = vector.shuffle %126, %130 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%147 = vector.shuffle %127, %131 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%148 = vector.shuffle %132, %140 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%149 = vector.shuffle %133, %141 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%150 = vector.shuffle %134, %142 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%151 = vector.shuffle %135, %143 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%152 = vector.shuffle %136, %144 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%153 = vector.shuffle %137, %145 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%154 = vector.shuffle %138, %146 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%155 = vector.shuffle %139, %147 [0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27] : vector<16xf16>, vector<16xf16>
%156 = vector.shuffle %132, %140 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%157 = vector.shuffle %133, %141 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%158 = vector.shuffle %134, %142 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%159 = vector.shuffle %135, %143 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%160 = vector.shuffle %136, %144 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%161 = vector.shuffle %137, %145 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%162 = vector.shuffle %138, %146 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%163 = vector.shuffle %139, %147 [4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31] : vector<16xf16>, vector<16xf16>
%164 = vector.insert %148, %cst [0] : vector<16xf16> into vector<16x16xf16>
%165 = vector.insert %149, %164 [1] : vector<16xf16> into vector<16x16xf16>
%166 = vector.insert %150, %165 [2] : vector<16xf16> into vector<16x16xf16>
%167 = vector.insert %151, %166 [3] : vector<16xf16> into vector<16x16xf16>
%168 = vector.insert %152, %167 [4] : vector<16xf16> into vector<16x16xf16>
%169 = vector.insert %153, %168 [5] : vector<16xf16> into vector<16x16xf16>
%170 = vector.insert %154, %169 [6] : vector<16xf16> into vector<16x16xf16>
%171 = vector.insert %155, %170 [7] : vector<16xf16> into vector<16x16xf16>
%172 = vector.insert %156, %171 [8] : vector<16xf16> into vector<16x16xf16>
%173 = vector.insert %157, %172 [9] : vector<16xf16> into vector<16x16xf16>
%174 = vector.insert %158, %173 [10] : vector<16xf16> into vector<16x16xf16>
%175 = vector.insert %159, %174 [11] : vector<16xf16> into vector<16x16xf16>
%176 = vector.insert %160, %175 [12] : vector<16xf16> into vector<16x16xf16>
%177 = vector.insert %161, %176 [13] : vector<16xf16> into vector<16x16xf16>
%178 = vector.insert %162, %177 [14] : vector<16xf16> into vector<16x16xf16>
%179 = vector.insert %163, %178 [15] : vector<16xf16> into vector<16x16xf16>
%180 = vector.shape_cast %179 : vector<16x16xf16> to vector<1x1x16x16x1xf16>
%181 = vector.extract %180[0, 0] : vector<16x16x1xf16> from vector<1x1x16x16x1xf16>
%subview_4 = memref.subview %subview[0, 0, 0, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%182 = vector.shape_cast %181 : vector<16x16x1xf16> to vector<16x16xf16>
%183 = vector.extract %182[0] : vector<16xf16> from vector<16x16xf16>
vector.store %183, %subview_4[%arg3, %arg4, %arg5, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%184 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg5)
%185 = vector.extract %182[1] : vector<16xf16> from vector<16x16xf16>
vector.store %185, %subview_4[%arg3, %arg4, %184, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%186 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg5)
%187 = vector.extract %182[2] : vector<16xf16> from vector<16x16xf16>
vector.store %187, %subview_4[%arg3, %arg4, %186, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%188 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg5)
%189 = vector.extract %182[3] : vector<16xf16> from vector<16x16xf16>
vector.store %189, %subview_4[%arg3, %arg4, %188, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%190 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg5)
%191 = vector.extract %182[4] : vector<16xf16> from vector<16x16xf16>
vector.store %191, %subview_4[%arg3, %arg4, %190, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%192 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg5)
%193 = vector.extract %182[5] : vector<16xf16> from vector<16x16xf16>
vector.store %193, %subview_4[%arg3, %arg4, %192, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%194 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg5)
%195 = vector.extract %182[6] : vector<16xf16> from vector<16x16xf16>
vector.store %195, %subview_4[%arg3, %arg4, %194, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%196 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg5)
%197 = vector.extract %182[7] : vector<16xf16> from vector<16x16xf16>
vector.store %197, %subview_4[%arg3, %arg4, %196, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%198 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg5)
%199 = vector.extract %182[8] : vector<16xf16> from vector<16x16xf16>
vector.store %199, %subview_4[%arg3, %arg4, %198, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%200 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg5)
%201 = vector.extract %182[9] : vector<16xf16> from vector<16x16xf16>
vector.store %201, %subview_4[%arg3, %arg4, %200, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%202 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg5)
%203 = vector.extract %182[10] : vector<16xf16> from vector<16x16xf16>
vector.store %203, %subview_4[%arg3, %arg4, %202, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%204 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg5)
%205 = vector.extract %182[11] : vector<16xf16> from vector<16x16xf16>
vector.store %205, %subview_4[%arg3, %arg4, %204, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%206 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg5)
%207 = vector.extract %182[12] : vector<16xf16> from vector<16x16xf16>
vector.store %207, %subview_4[%arg3, %arg4, %206, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%208 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg5)
%209 = vector.extract %182[13] : vector<16xf16> from vector<16x16xf16>
vector.store %209, %subview_4[%arg3, %arg4, %208, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%210 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg5)
%211 = vector.extract %182[14] : vector<16xf16> from vector<16x16xf16>
vector.store %211, %subview_4[%arg3, %arg4, %210, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%212 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg5)
%213 = vector.extract %182[15] : vector<16xf16> from vector<16x16xf16>
vector.store %213, %subview_4[%arg3, %arg4, %212, %c0] : memref<?x4x64x16xf16, strided<[27648000, 51200, 16, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
}
}
}
}
}
}
return
}
// -----// IR Dump After Canonicalizer (canonicalize) //----- //
func.func @broadcast_pack_kernel_dispatch_0_broadcast_Dx8640x3200_f16_pack() attributes {translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} {
%c15 = arith.constant 15 : index
%c14 = arith.constant 14 : index
%c13 = arith.constant 13 : index
%c12 = arith.constant 12 : index
%c11 = arith.constant 11 : index
%c10 = arith.constant 10 : index
%c9 = arith.constant 9 : index
%c8 = arith.constant 8 : index
%c7 = arith.constant 7 : index
%c6 = arith.constant 6 : index
%c5 = arith.constant 5 : index
%c3 = arith.constant 3 : index
%c2 = arith.constant 2 : index
%c32_i64 = arith.constant 32 : i64
%c0 = arith.constant 0 : index
%c540 = arith.constant 540 : index
%c3200 = arith.constant 3200 : index
%c1 = arith.constant 1 : index
%c4 = arith.constant 4 : index
%c64 = arith.constant 64 : index
%c16 = arith.constant 16 : index
%alloca = memref.alloca() {alignment = 64 : i64} : memref<1x16x16xf16>
%0 = hal.interface.constant.load[0] : i32
%1 = hal.interface.constant.load[1] : i32
%2 = arith.extui %0 : i32 to i64
%3 = arith.extui %1 : i32 to i64
%4 = arith.shli %3, %c32_i64 : i64
%5 = arith.ori %2, %4 : i64
%6 = arith.index_castui %5 : i64 to index
%7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>>
%8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>{%6}
memref.assume_alignment %8, 64 : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>>
%workgroup_id_x = hal.interface.workgroup.id[0] : index
%workgroup_count_x = hal.interface.workgroup.count[0] : index
%workgroup_id_y = hal.interface.workgroup.id[1] : index
%workgroup_count_y = hal.interface.workgroup.count[1] : index
%workgroup_id_z = hal.interface.workgroup.id[2] : index
%workgroup_count_z = hal.interface.workgroup.count[2] : index
%9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z]
%10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z]
%11 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_id_y]
%12 = affine.apply affine_map<()[s0] -> (s0 * 4)>()[%workgroup_count_y]
%13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x]
%14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x]
scf.for %arg0 = %9 to %6 step %10 {
%15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6]
scf.for %arg1 = %11 to %c540 step %12 {
%16 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg1)
scf.for %arg2 = %13 to %c3200 step %14 {
%subview = memref.subview %8[%arg0, %arg1, %arg2, 0, 0] [%15, 4, 64, 16, 1] [1, 1, 1, 1, 1] : memref<?x540x3200x16x1xf16, #hal.descriptor_type<storage_buffer>> to memref<?x4x64x16x1xf16, strided<[27648000, 51200, 16, 1, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
%subview_0 = memref.subview %7[%16, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>
scf.for %arg3 = %c0 to %15 step %c1 {
scf.for %arg4 = %c0 to %c4 step %c1 {
scf.for %arg5 = %c0 to %c64 step %c16 {
%17 = affine.apply affine_map<(d0) -> (d0 * 16)>(%arg4)
%18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%19 = affine.apply affine_map<(d0) -> (d0 * 16 + 1)>(%arg4)
%20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%21 = affine.apply affine_map<(d0) -> (d0 * 16 + 2)>(%arg4)
%22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%23 = affine.apply affine_map<(d0) -> (d0 * 16 + 3)>(%arg4)
%24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%25 = affine.apply affine_map<(d0) -> (d0 * 16 + 4)>(%arg4)
%26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%27 = affine.apply affine_map<(d0) -> (d0 * 16 + 5)>(%arg4)
%28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%29 = affine.apply affine_map<(d0) -> (d0 * 16 + 6)>(%arg4)
%30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%31 = affine.apply affine_map<(d0) -> (d0 * 16 + 7)>(%arg4)
%32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%33 = affine.apply affine_map<(d0) -> (d0 * 16 + 8)>(%arg4)
%34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%35 = affine.apply affine_map<(d0) -> (d0 * 16 + 9)>(%arg4)
%36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%37 = affine.apply affine_map<(d0) -> (d0 * 16 + 10)>(%arg4)
%38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%39 = affine.apply affine_map<(d0) -> (d0 * 16 + 11)>(%arg4)
%40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%41 = affine.apply affine_map<(d0) -> (d0 * 16 + 12)>(%arg4)
%42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%43 = affine.apply affine_map<(d0) -> (d0 * 16 + 13)>(%arg4)
%44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%45 = affine.apply affine_map<(d0) -> (d0 * 16 + 14)>(%arg4)
%46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%47 = affine.apply affine_map<(d0) -> (d0 * 16 + 15)>(%arg4)
%48 = vector.load %subview_0[%47, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16>
%subview_1 = memref.subview %alloca[0, 0, 0] [1, 16, 16] [1, 1, 1] : memref<1x16x16xf16> to memref<16x16xf16>
vector.store %18, %subview_1[%c0, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %20, %subview_1[%c1, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %22, %subview_1[%c2, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %24, %subview_1[%c3, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %26, %subview_1[%c4, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %28, %subview_1[%c5, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %30, %subview_1[%c6, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %32, %subview_1[%c7, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %34, %subview_1[%c8, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %36, %subview_1[%c9, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %38, %subview_1[%c10, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %40, %subview_1[%c11, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %42, %subview_1[%c12, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %44, %subview_1[%c13, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %46, %subview_1[%c14, %c0] : memref<16x16xf16>, vector<16xf16>
vector.store %48, %subview_1[%c15, %c0] : memref<16x16xf16>, vector<16xf16>
%expand_shape = memref.expand_shape %alloca [[0], [1, 2], [3, 4]] : memref<1x16x16xf16> into memref<1x1x16x16x1xf16>
%subview_2 = memref.subview %expand_shape[0, 0, 0, 0, 0] [1, 1, 16, 16, 1] [1, 1, 1, 1, 1] : memref<1x1x16x16x1xf16> to memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>>
%subview_3 = memref.subview %subview_2[0, 0, 0, 0] [1, 1, 16, 16] [1, 1, 1, 1] : memref<1x1x16x16xf16, strided<[256, 256, 16, 1]>> to memref<16x16xf16>
%49 = vector.load %subview_3[%c0, %c0] : memref<16x16xf16>, vector<16xf16>
%50 = vector.load %subview_3[%c1, %c0] : memref<16x16xf16>, vector<16xf16>
%51 = vector.load %subview_3[%c2, %c0] : memref<16x16xf16>, vector<16xf16>
%52 = vector.load %subview_3[%c3, %c0] : memref<16x16xf16>, vector<16xf16>
%53 = vector.load %subview_3[%c4, %c0] : memref<16x16xf16>, vector<16xf16>
%54 = vector.load %subview_3[%c5, %c0] : memref<16x16xf16>, vector<16xf16>
%55 = vector.load %subview_3[%c6, %c0] : memref<16x16xf16>, vector<16xf16>
%56 = vector.load %subview_3[%c7, %c0] : memref<16x16xf16>, vector<16xf16>
%57 = vector.load %subview_3[%c8, %c0] : memref<16x16xf16>, vector<16xf16>
%58 = vector.load %subview_3[%c9, %c0] : memref<16x16xf16>, vector<16xf16>
%59 = vector.load %subview_3[%c10, %c0] : memref<16x16xf16>, vector<16xf16>
%60 = vector.load %subview_3[%c11, %c0] : memref<16x16xf16>, vector<16xf16>
%61 = vector.load %subview_3[%c12, %c0] : memref<16x16xf16>, vector<16xf16>
%62 = vector.load %subview_3[%c13, %c0] : memref<16x16xf16>, vector<16xf16>
%63 = vector.load %subview_3[%c14, %c0] : memref<16x16xf16>, vector<16xf16>
%64 = vector.load %subview_3[%c15, %c0] : memref<16x16xf16>, vector<16xf16>
%65 = vector.shuffle %49, %50 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%66 = vector.shuffle %49, %50 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%67 = vector.shuffle %51, %52 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%68 = vector.shuffle %51, %52 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%69 = vector.shuffle %53, %54 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%70 = vector.shuffle %53, %54 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%71 = vector.shuffle %55, %56 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%72 = vector.shuffle %55, %56 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%73 = vector.shuffle %57, %58 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%74 = vector.shuffle %57, %58 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%75 = vector.shuffle %59, %60 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%76 = vector.shuffle %59, %60 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%77 = vector.shuffle %61, %62 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%78 = vector.shuffle %61, %62 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%79 = vector.shuffle %63, %64 [0, 16, 1, 17, 4, 20, 5, 21, 8, 24, 9, 25, 12, 28, 13, 29] : vector<16xf16>, vector<16xf16>
%80 = vector.shuffle %63, %64 [2, 18, 3, 19, 6, 22, 7, 23, 10, 26, 11, 27, 14, 30, 15, 31] : vector<16xf16>, vector<16xf16>
%81 = vector.shuffle %65, %67 [0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29] : vector<16xf16>, vector<16xf16>
%82 = vector.shuffle %65, %67 [2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31] : vector<16xf16>, vector<16xf16>
%83 = vector.shuffle %66, %68 [0, 1, 16, 17, 4,
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment