Created
April 24, 2024 13:02
-
-
Save pashu123/8d73e2ffd6cd9c500bdf81542a43b87f to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // -----// IR Dump After AssignTargetDevicesPass (iree-hal-assign-target-devices) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| func.func @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16> | |
| %1 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| return %2 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After AutoInputConversionPipeline (iree-auto-input-conversion) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| func.func @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16> | |
| %1 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| return %2 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After IREEImportPublic (iree-import-public) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16> | |
| %1 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %2 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After ImportMLProgram (iree-import-ml-program) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16> | |
| %1 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %2 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After SanitizeModuleNames (iree-sanitize-module-names) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x540x3200x16x1xf16> | |
| %1 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %2 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%1 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %2 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After ConvertMeshToFlow (iree-convert-mesh-to-flow) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %1 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%0 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %1 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After mlir::iree_compiler::IREE::ABI::ConvertStreamableOpsPass (iree-abi-convert-streamable-ops) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %1 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%0 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %1 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After mlir::iree_compiler::IREE::ABI::WrapEntryPointsPass (iree-abi-wrap-entry-points) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[1] : index | |
| %2 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<?x?x3200xf32>{%0, %1} | |
| %3 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %4 = util.call @_broadcast(%2, %3) : (tensor<?x?x3200xf32>, tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> | |
| %c0 = arith.constant 0 : index | |
| %dim = tensor.dim %4, %c0 : tensor<?x8640x3200xf16> | |
| %5 = hal.tensor.export %4 "output0" : tensor<?x8640x3200xf16>{%dim} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| util.func private @_broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %1 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%0 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %1 : tensor<?x8640x3200xf16> | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func private @_broadcast(%arg0: tensor<?x?x3200xf32>, %arg1: tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> { | |
| %c0 = arith.constant 0 : index | |
| %dim = tensor.dim %arg0, %c0 : tensor<?x?x3200xf32> | |
| %0 = tensor.empty(%dim) : tensor<?x8640x3200xf16> | |
| %1 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%arg1 : tensor<8640x3200xf16>) outs(%0 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| util.return %1 : tensor<?x8640x3200xf16> | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[1] : index | |
| %2 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<?x?x3200xf32>{%0, %1} | |
| %3 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %4 = util.call @_broadcast(%2, %3) : (tensor<?x?x3200xf32>, tensor<8640x3200xf16>) -> tensor<?x8640x3200xf16> | |
| %dim = tensor.dim %4, %c0 : tensor<?x8640x3200xf16> | |
| %5 = hal.tensor.export %4 "output0" : tensor<?x8640x3200xf16>{%dim} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Inliner (inline) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SymbolDCE (symbol-dce) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After DemoteF64ToF32 (iree-util-demote-f64-to-f32) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After RemoveZeroExtentTensors (iree-global-opt-remove-zero-extent-tensors) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After DetachElementwiseFromNamedOps (iree-global-opt-detach-elementwise-from-named-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After LinalgNamedOpConversionPass (linalg-named-op-conversion) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Convert1X1FilterConv2DToMatmul (iree-global-opt-convert-1x1-filter-conv2d-to-matmul) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After EraseUnusedLinalgOperands (iree-global-opt-erase-unused-linalg-operands) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After ExpandTensorShapes (iree-global-opt-expand-tensor-shapes) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After ConvertElementwiseToLinalgPass (convert-elementwise-to-linalg) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After RaiseSpecialOps (iree-global-opt-raise-special-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After DecomposeConcat (iree-global-opt-decompose-concat) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After GeneralizeLinalgNamedOps (iree-global-opt-generalize-linalg-named-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FoldUnitExtentDims (iree-flow-fold-unit-extent-dims) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FuseDequantizationMatmul (iree-global-opt-fuse-dequantization-matmul) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SetEncoding (iree-global-opt-set-encoding) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CPUMaterializeUpperBoundTileSize (iree-codegen-cpu-materialize-upper-bound-tile-size) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CPUMaterializeEncoding (iree-codegen-cpu-materialize-encoding) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After MaterializeHomogeneousEncodings (iree-global-opt-materialize-homogeneous-encodings) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After SimplifyPackUnpack (iree-global-opt-simplify-pack-unpack) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After GeneralizeLinalgNamedOps (iree-global-opt-generalize-linalg-named-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After GlobalLoopInvariantCodeMotion (iree-global-opt-loop-invariant-code-motion) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After HoistIntoGlobals (iree-util-hoist-into-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After JitGlobals (iree-consteval-jit-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FixedPointIterator (iree-util-fixed-point-iterator) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After RaiseSpecialOps (iree-global-opt-raise-special-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After VerifyInputLegality (iree-verify-input-legality) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After InjectTensorTracing (iree-flow-inject-tensor-tracing) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After TensorPadToTensorInsertSlice (iree-flow-tensor-pad-to-tensor-insert-slice) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After InterchangeGenericOps (iree-flow-interchange-generic-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ResolveShapedTypeResultDims (resolve-shaped-type-result-dims) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FusionOfTensorOps (iree-flow-fusion-of-tensor-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SplitReduction (iree-flow-split-reduction-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After InterchangeGenericOps (iree-flow-interchange-generic-ops) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FormScalarDispatches (iree-flow-form-scalar-dispatches) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FormDispatchRegions (iree-flow-form-dispatch-regions) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = flow.dispatch.region -> (tensor<?x8640x3200xf16>{%0}) { | |
| %5 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%2 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.return %5 : tensor<?x8640x3200xf16> | |
| } | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CloneProducersIntoDispatchRegions (iree-flow-clone-producers-into-dispatch-regions) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = flow.dispatch.region -> (tensor<?x8640x3200xf16>{%0}) { | |
| %5 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.return %6 : tensor<?x8640x3200xf16> | |
| } | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CollapseDimensions (iree-flow-collapse-dimensions) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %3 = flow.dispatch.region -> (tensor<?x8640x3200xf16>{%0}) { | |
| %5 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %6 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%1 : tensor<8640x3200xf16>) outs(%5 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.return %6 : tensor<?x8640x3200xf16> | |
| } | |
| %4 = hal.tensor.export %3 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After FormDispatchWorkgroups (iree-flow-form-dispatch-workgroups) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} = | |
| (%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %4 = flow.dispatch.workload.ordinal %arg3, 0 : index | |
| %5 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %6 = tensor.empty(%4) : tensor<?x8640x3200xf16> | |
| %7 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%5 : tensor<8640x3200xf16>) outs(%6 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %7, %arg4, offsets = [0, 0, 0], sizes = [%4, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| flow.return | |
| } count(%arg2: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CaptureDynamicDims (iree-flow-capture-dynamic-dims) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} = | |
| (%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %4 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%arg3} | |
| %5 = flow.dispatch.workload.ordinal %arg3, 0 : index | |
| %6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %7 = tensor.empty(%5) : tensor<?x8640x3200xf16> | |
| %8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %8, %4, offsets = [0, 0, 0], sizes = [%5, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%5} | |
| flow.return | |
| } count(%arg2: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} = | |
| (%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %4 = flow.dispatch.workload.ordinal %arg3, 0 : index | |
| %5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| %6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %7 = tensor.empty(%4) : tensor<?x8640x3200xf16> | |
| %8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %8, %5, offsets = [0, 0, 0], sizes = [%4, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| flow.return | |
| } count(%arg2: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} = | |
| (%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %4 = flow.dispatch.workload.ordinal %arg3, 0 : index | |
| %5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| %6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %7 = tensor.empty(%4) : tensor<?x8640x3200xf16> | |
| %8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %8, %5, offsets = [0, 0, 0], sizes = [%4, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| flow.return | |
| } count(%arg2: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After InitializeEmptyTensors (iree-flow-initialize-empty-tensors) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} = | |
| (%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %4 = flow.dispatch.workload.ordinal %arg3, 0 : index | |
| %5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| %6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %7 = tensor.empty(%4) : tensor<?x8640x3200xf16> | |
| %8 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %8, %5, offsets = [0, 0, 0], sizes = [%4, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| flow.return | |
| } count(%arg2: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After OutlineDispatchExterns (iree-flow-outline-dispatch-externs) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch.workgroups[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} = | |
| (%arg2: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg3: index, %arg4: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %4 = flow.dispatch.workload.ordinal %arg3, 0 : index | |
| %5 = flow.dispatch.tie_shape %arg4 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| %6 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %7 = tensor.empty(%4) : tensor<?x8640x3200xf16> | |
| %8 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%6 : tensor<8640x3200xf16>) outs(%7 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %8, %5, offsets = [0, 0, 0], sizes = [%4, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%4} | |
| flow.return | |
| } count(%arg2: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg2 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After OutlineDispatchRegions (iree-flow-outline-dispatch-regions) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After AnnotateDispatches (iree-flow-annotate-dispatches) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After StripDebugOps (iree-util-strip-debug-ops) //----- // | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After DeduplicateExecutables (iree-flow-deduplicate-executables) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After InjectTensorTracing (iree-flow-inject-tensor-tracing) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CleanupTensorShapes (iree-flow-cleanup-tensor-shapes) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SymbolDCE (symbol-dce) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyInputPass (iree-stream-verify-input) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After OutlineConstants (iree-util-outline-constants) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| flow.executable private @broadcast_dispatch_0 { | |
| flow.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| flow.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>>, %arg1: index, %arg2: !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>) { | |
| %0 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %1 = flow.dispatch.tie_shape %arg2 : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| %2 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %3 = tensor.empty(%0) : tensor<?x8640x3200xf16> | |
| %4 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%2 : tensor<8640x3200xf16>) outs(%3 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %4, %1, offsets = [0, 0, 0], sizes = [%0, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%0} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<8640x3200xf16> | |
| %2 = flow.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1, %0) : (tensor<8640x3200xf16>, index) -> tensor<?x8640x3200xf16>{%0} | |
| %3 = hal.tensor.export %2 "output0" : tensor<?x8640x3200xf16>{%0} -> !hal.buffer_view | |
| util.return %3 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After ConvertToStreamPass (iree-stream-conversion) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| %c8640 = arith.constant 8640 : index | |
| %c3200 = arith.constant 3200 : index | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %c0 = arith.constant 0 : index | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyLoweringToTensorsPass (iree-stream-verify-lowering-to-tensors) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| %c8640 = arith.constant 8640 : index | |
| %c3200 = arith.constant 3200 : index | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %c0 = arith.constant 0 : index | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CombineInitializers (iree-util-combine-initializers) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.sizeof tensor<8640x3200xf16> : index | |
| %2 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%1} | |
| %3 = stream.async.transfer %2 : !stream.resource<external>{%1} -> !stream.resource<*>{%1} | |
| %4 = stream.tensor.sizeof tensor<?x8640x3200xf16>{%0} : index | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%3[%c0 to %1 for %1], %0) : (!stream.resource<*>{%1}, index) -> !stream.resource<*>{%4} | |
| %6 = stream.async.transfer %5 : !stream.resource<*>{%4} -> !stream.resource<external>{%4} | |
| %7 = stream.tensor.export %6 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%4} -> !hal.buffer_view | |
| util.return %7 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After EncodeDeviceTensorsPass (iree-stream-encode-device-tensors) //----- // | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After EncodeHostTensorsPass (iree-stream-encode-host-tensors) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyLoweringToAsyncResourcesPass (iree-stream-verify-lowering-to-async-resources) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After MaterializeCopyOnWritePass (iree-stream-materialize-copy-on-write) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ElideAsyncCopiesPass (iree-stream-elide-async-copies) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After EmplaceAllocationsPass (iree-stream-emplace-allocations) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = stream.async.transfer %1 : !stream.resource<external>{%c55296000} -> !stream.resource<*>{%c55296000} | |
| %3 = arith.muli %0, %c55296000 : index | |
| %4 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<*>{%c55296000}, index) -> !stream.resource<*>{%3} | |
| %5 = stream.async.transfer %4 : !stream.resource<*>{%3} -> !stream.resource<external>{%3} | |
| %6 = stream.tensor.export %5 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%3} -> !hal.buffer_view | |
| util.return %6 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After RefineUsagePass (iree-stream-refine-usage) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyAsyncAccessRangesPass (iree-stream-verify-async-access-ranges) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%1[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After ScheduleExecutionPass (iree-stream-schedule-execution) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ScheduleConcurrencyPass (iree-stream-schedule-concurrency) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After PropagateTimepointsPass (iree-stream-propagate-timepoints) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.timepoint.immediate => !stream.timepoint | |
| %results, %result_timepoint = stream.async.execute await(%3) => with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %6 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %6 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After MaterializeBuiltinsPass (iree-stream-materialize-builtins) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %3 = stream.timepoint.immediate => !stream.timepoint | |
| %results, %result_timepoint = stream.async.execute await(%3) => with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %6 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %6 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyLoweringToAsyncPass (iree-stream-verify-lowering-to-async) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %results, %result_timepoint = stream.async.execute with(%1 as %arg2: !stream.resource<external>{%c55296000}) -> !stream.resource<external>{%2} { | |
| %5 = stream.async.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%arg2[%c0 to %c55296000 for %c55296000], %0) : (!stream.resource<external>{%c55296000}, index) -> !stream.resource<external>{%2} | |
| stream.yield %5 : !stream.resource<external>{%2} | |
| } => !stream.timepoint | |
| %3 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%2} | |
| %4 = stream.tensor.export %3 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %4 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After ScheduleAllocationPass (iree-stream-schedule-allocation) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %c0_0 = arith.constant 0 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After PackConstantsPass (iree-stream-pack-constants) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %c0_0 = arith.constant 0 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After LayoutSlicesPass (iree-stream-layout-slices) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %c0_0 = arith.constant 0 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After PropagateSubranges (iree-util-propagate-subranges) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %c0_0 = arith.constant 0 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyLoweringToCmdPass (iree-stream-verify-lowering-to-cmd) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After SCFToControlFlow (convert-scf-to-cf) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After ElideTimepointsPass (iree-stream-elide-timepoints) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu], iree.fixedpoint.iteration = 0 : index} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FixedPointIterator (iree-util-fixed-point-iterator) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: index, %arg2: !stream.binding) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg1, 0 : index | |
| %2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%0 : index) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseDispatchBindingsPass (iree-stream-fuse-dispatch-bindings) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: index, %arg3: index, %arg4: index) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%arg2] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg4, 0 : index | |
| %2 = stream.binding.subspan %arg1[%arg3] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %c0_0 = arith.constant 0 : index | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0, %c0, %0 : index, index, index) { | |
| ro %arg2[%c0_0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After AnnotateDispatchArgumentsPass (iree-stream-annotate-dispatch-arguments) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: index {stream.values = [0 : index]}, %arg3: index {stream.values = [0 : index]}, %arg4: index) { | |
| %c0 = arith.constant 0 : index | |
| %0 = stream.binding.subspan %arg0[%arg2] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %1 = flow.dispatch.workload.ordinal %arg4, 0 : index | |
| %2 = stream.binding.subspan %arg1[%arg3] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| %3 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %4 = tensor.empty(%1) : tensor<?x8640x3200xf16> | |
| %5 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%3 : tensor<8640x3200xf16>) outs(%4 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %5, %2, offsets = [0, 0, 0], sizes = [%1, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%1} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %c0_0 = arith.constant 0 : index | |
| %3 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0, %c0, %0 : index, index, index) { | |
| ro %arg2[%c0_0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %4 = stream.timepoint.await %3 => %result : !stream.resource<external>{%2} | |
| %5 = stream.tensor.export %4 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %5 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After PackDispatchOperandsPass (iree-stream-pack-dispatch-operands) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) { | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %c32_i64 = arith.constant 32 : i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index | |
| %5 = arith.extui %arg4 : i32 to i64 | |
| %6 = arith.extui %arg5 : i32 to i64 | |
| %c32_i64_0 = arith.constant 32 : i64 | |
| %7 = arith.shli %6, %c32_i64_0 : i64 | |
| %8 = arith.ori %5, %7 : i64 | |
| %9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index | |
| %10 = arith.extui %arg6 : i32 to i64 | |
| %11 = arith.extui %arg7 : i32 to i64 | |
| %c32_i64_1 = arith.constant 32 : i64 | |
| %12 = arith.shli %11, %c32_i64_1 : i64 | |
| %13 = arith.ori %10, %12 : i64 | |
| %14 = arith.index_castui %13 : i64 to index | |
| %c0 = arith.constant 0 : index | |
| %15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %16 = flow.dispatch.workload.ordinal %14, 0 : index | |
| %17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| %18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %19 = tensor.empty(%16) : tensor<?x8640x3200xf16> | |
| %20 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%19 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %20, %17, offsets = [0, 0, 0], sizes = [%16, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %c0_0 = arith.constant 0 : index | |
| %c0_i64 = arith.constant 0 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i64_1 = arith.constant 0 : i64 | |
| %c0_i32_2 = arith.constant 0 : i32 | |
| %c0_i64_3 = arith.constant 0 : i64 | |
| %c0_i32_4 = arith.constant 0 : i32 | |
| %c32_i64_5 = arith.constant 32 : i64 | |
| %c0_i64_6 = arith.constant 0 : i64 | |
| %c0_i32_7 = arith.constant 0 : i32 | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %c32_i64_8 = arith.constant 32 : i64 | |
| %5 = arith.shrui %3, %c32_i64_8 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32_2, %c0_i32_4, %c0_i32_7, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0_0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0_0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index | |
| %5 = arith.extui %arg4 : i32 to i64 | |
| %6 = arith.extui %arg5 : i32 to i64 | |
| %7 = arith.shli %6, %c32_i64 : i64 | |
| %8 = arith.ori %5, %7 : i64 | |
| %9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index | |
| %10 = arith.extui %arg6 : i32 to i64 | |
| %11 = arith.extui %arg7 : i32 to i64 | |
| %12 = arith.shli %11, %c32_i64 : i64 | |
| %13 = arith.ori %10, %12 : i64 | |
| %14 = arith.index_castui %13 : i64 to index | |
| %15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %16 = flow.dispatch.workload.ordinal %14, 0 : index | |
| %17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| %18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %19 = tensor.empty(%16) : tensor<?x8640x3200xf16> | |
| %20 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%19 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %20, %17, offsets = [0, 0, 0], sizes = [%16, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index | |
| %5 = arith.extui %arg4 : i32 to i64 | |
| %6 = arith.extui %arg5 : i32 to i64 | |
| %7 = arith.shli %6, %c32_i64 : i64 | |
| %8 = arith.ori %5, %7 : i64 | |
| %9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index | |
| %10 = arith.extui %arg6 : i32 to i64 | |
| %11 = arith.extui %arg7 : i32 to i64 | |
| %12 = arith.shli %11, %c32_i64 : i64 | |
| %13 = arith.ori %10, %12 : i64 | |
| %14 = arith.index_castui %13 : i64 to index | |
| %15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %16 = flow.dispatch.workload.ordinal %14, 0 : index | |
| %17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| %18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %19 = tensor.empty(%16) : tensor<?x8640x3200xf16> | |
| %20 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%19 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %20, %17, offsets = [0, 0, 0], sizes = [%16, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index | |
| %5 = arith.extui %arg4 : i32 to i64 | |
| %6 = arith.extui %arg5 : i32 to i64 | |
| %7 = arith.shli %6, %c32_i64 : i64 | |
| %8 = arith.ori %5, %7 : i64 | |
| %9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index | |
| %10 = arith.extui %arg6 : i32 to i64 | |
| %11 = arith.extui %arg7 : i32 to i64 | |
| %12 = arith.shli %11, %c32_i64 : i64 | |
| %13 = arith.ori %10, %12 : i64 | |
| %14 = arith.index_castui %13 : i64 to index | |
| %15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %16 = flow.dispatch.workload.ordinal %14, 0 : index | |
| %17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| %18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %19 = tensor.empty(%16) : tensor<?x8640x3200xf16> | |
| %20 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%19 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %20, %17, offsets = [0, 0, 0], sizes = [%16, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32, %arg4: i32, %arg5: i32, %arg6: i32, %arg7: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index | |
| %5 = arith.extui %arg4 : i32 to i64 | |
| %6 = arith.extui %arg5 : i32 to i64 | |
| %7 = arith.shli %6, %c32_i64 : i64 | |
| %8 = arith.ori %5, %7 : i64 | |
| %9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index | |
| %10 = arith.extui %arg6 : i32 to i64 | |
| %11 = arith.extui %arg7 : i32 to i64 | |
| %12 = arith.shli %11, %c32_i64 : i64 | |
| %13 = arith.ori %10, %12 : i64 | |
| %14 = arith.index_castui %13 : i64 to index | |
| %15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %16 = flow.dispatch.workload.ordinal %14, 0 : index | |
| %17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| %18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %19 = tensor.empty(%16) : tensor<?x8640x3200xf16> | |
| %20 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%19 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %20, %17, offsets = [0, 0, 0], sizes = [%16, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%c0_i32, %c0_i32, %c0_i32, %c0_i32, %4, %6 : i32, i32, i32, i32, i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldUniformOperandsPass (iree-stream-fold-uniform-operands) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = arith.extui %c0_i32 : i32 to i64 | |
| %1 = arith.extui %c0_i32 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 {stream.values = [0 : index]} : i64 to index | |
| %5 = arith.extui %c0_i32 : i32 to i64 | |
| %6 = arith.extui %c0_i32 : i32 to i64 | |
| %7 = arith.shli %6, %c32_i64 : i64 | |
| %8 = arith.ori %5, %7 : i64 | |
| %9 = arith.index_castui %8 {stream.values = [0 : index]} : i64 to index | |
| %10 = arith.extui %arg2 : i32 to i64 | |
| %11 = arith.extui %arg3 : i32 to i64 | |
| %12 = arith.shli %11, %c32_i64 : i64 | |
| %13 = arith.ori %10, %12 : i64 | |
| %14 = arith.index_castui %13 : i64 to index | |
| %15 = stream.binding.subspan %arg0[%4] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %16 = flow.dispatch.workload.ordinal %14, 0 : index | |
| %17 = stream.binding.subspan %arg1[%9] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| %18 = flow.dispatch.tensor.load %15, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %19 = tensor.empty(%16) : tensor<?x8640x3200xf16> | |
| %20 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%18 : tensor<8640x3200xf16>) outs(%19 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %20, %17, offsets = [0, 0, 0], sizes = [%16, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%16} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0_i32 = arith.constant 0 : i32 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After IPO (iree-util-ipo) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After SymbolDCE (symbol-dce) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FoldGlobals (iree-util-fold-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After FuseGlobals (iree-util-fuse-globals) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After AssignTargetDevicesPass (iree-hal-assign-target-devices) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After VerifyTargetEnvironmentPass (iree-hal-verify-target-environment) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| stream.executable private @broadcast_dispatch_0 { | |
| stream.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 workgroups(%arg0: index) -> (index, index, index) { | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg0 | |
| stream.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16(%arg0: !stream.binding {stream.alignment = 64 : index}, %arg1: !stream.binding {stream.alignment = 64 : index}, %arg2: i32, %arg3: i32) { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c0 = arith.constant 0 : index | |
| %0 = arith.extui %arg2 : i32 to i64 | |
| %1 = arith.extui %arg3 : i32 to i64 | |
| %2 = arith.shli %1, %c32_i64 : i64 | |
| %3 = arith.ori %0, %2 : i64 | |
| %4 = arith.index_castui %3 : i64 to index | |
| %5 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %6 = flow.dispatch.workload.ordinal %4, 0 : index | |
| %7 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %8 = flow.dispatch.tensor.load %5, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %9 = tensor.empty(%6) : tensor<?x8640x3200xf16> | |
| %10 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%8 : tensor<8640x3200xf16>) outs(%9 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %10, %7, offsets = [0, 0, 0], sizes = [%6, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| return | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After MaterializeInterfacesPass (iree-hal-materialize-interfaces) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #pipeline_layout = #hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| hal.executable private @broadcast_dispatch_0 { | |
| hal.executable.variant public @embedded_elf_x86_64 target(#executable_target_embedded_elf_x86_64_) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#pipeline_layout) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1 | |
| hal.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@embedded_elf_x86_64::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After PruneExecutablesPass (iree-hal-prune-executables) //----- // | |
| #executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
| #map = affine_map<(d0, d1, d2) -> (d1, d2)> | |
| #map1 = affine_map<(d0, d1, d2) -> (d0, d1, d2)> | |
| #pipeline_layout = #hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]> | |
| #device_target_llvm_cpu = #hal.device.target<"llvm-cpu", [#executable_target_embedded_elf_x86_64_]> | |
| module attributes {hal.device.targets = [#device_target_llvm_cpu]} { | |
| hal.executable private @broadcast_dispatch_0 { | |
| hal.executable.variant public @embedded_elf_x86_64 target(#executable_target_embedded_elf_x86_64_) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#pipeline_layout) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1 | |
| hal.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| } | |
| } | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@embedded_elf_x86_64::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| } | |
| // -----// IR Dump After CPUMaterializeUpperBoundTileSize (iree-codegen-cpu-materialize-upper-bound-tile-size) //----- // | |
| util.func public @broadcast(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @broadcast(%input0: tensor<?x?x3200xf32>, %input1: tensor<8640x3200xf16>) -> (%output0: tensor<?x8640x3200xf16>)"}} { | |
| %c32_i64 = arith.constant 32 : i64 | |
| %c55296000 = arith.constant 55296000 : index | |
| %c0 = arith.constant 0 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %0 = hal.buffer_view.dim<%arg0 : !hal.buffer_view>[0] : index | |
| %element_type_f16 = hal.element_type<f16> : i32 | |
| %dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
| hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c8640, %c3200]) type(%element_type_f16) encoding(%dense_row_major) | |
| %1 = stream.tensor.import %arg1 : !hal.buffer_view -> tensor<8640x3200xf16> in !stream.resource<external>{%c55296000} | |
| %2 = arith.muli %0, %c55296000 : index | |
| %result, %result_timepoint = stream.resource.alloca uninitialized : !stream.resource<external>{%2} => !stream.timepoint | |
| %3 = arith.index_castui %0 : index to i64 | |
| %4 = arith.trunci %3 : i64 to i32 | |
| %5 = arith.shrui %3, %c32_i64 : i64 | |
| %6 = arith.trunci %5 : i64 to i32 | |
| %7 = stream.cmd.execute await(%result_timepoint) => with(%1 as %arg2: !stream.resource<external>{%c55296000}, %result as %arg3: !stream.resource<external>{%2}) { | |
| stream.cmd.dispatch @broadcast_dispatch_0::@embedded_elf_x86_64::@broadcast_dispatch_0_generic_Dx8640x3200_f16[%0](%4, %6 : i32, i32) { | |
| ro %arg2[%c0 for %c55296000] : !stream.resource<external>{%c55296000}, | |
| wo %arg3[%c0 for %2] : !stream.resource<external>{%2} | |
| } | |
| } => !stream.timepoint | |
| %8 = stream.timepoint.await %7 => %result : !stream.resource<external>{%2} | |
| %9 = stream.tensor.export %8 : tensor<?x8640x3200xf16>{%0} in !stream.resource<external>{%2} -> !hal.buffer_view | |
| util.return %9 : !hal.buffer_view | |
| } | |
| // -----// IR Dump After TypePropagation (iree-codegen-type-propagation) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| // -----// IR Dump After BubbleUpOrdinalOps (iree-codegen-bubble-up-ordinal-ops) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| // -----// IR Dump After BufferizeCopyOnlyDispatches (iree-codegen-bufferize-copy-only-dispatches) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| // -----// IR Dump After DecomposeSoftmax (iree-codegen-decompose-softmax) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| // -----// IR Dump After MaterializeUserConfigs (iree-codegen-materialize-user-configs) //----- // | |
| hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>]} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1 | |
| hal.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After RematerializeParallelOps (iree-codegen-rematerialize-parallel-ops) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| // -----// IR Dump After ExpandF16OpToF32 (iree-llvmcpu-expand-f16-op-to-f32) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| // -----// IR Dump After CPUMaterializeEncoding (iree-codegen-cpu-materialize-encoding) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| // -----// IR Dump After EraseHALDescriptorTypeFromMemRef (iree-codegen-erase-hal-descriptor-type-from-memref) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| // -----// IR Dump After LLVMCPUSelectLoweringStrategy (iree-llvmcpu-select-lowering-strategy) //----- // | |
| hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>], translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1 | |
| hal.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After ConfigureTargetExecutableVariantsPass (iree-hal-configure-target-executable-variants) //----- // | |
| hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>], translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1 | |
| hal.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After ConfigureExecutablesPass (iree-hal-configure-executables) //----- // | |
| hal.executable private @broadcast_dispatch_0 { | |
| hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>], translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %x, %y, %z = flow.dispatch.workgroup_count_from_slice %arg1 | |
| hal.return %x, %y, %z : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = flow.dispatch.workload.ordinal %6, 0 : index | |
| %9 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| %10 = flow.dispatch.tensor.load %7, offsets = [0, 0], sizes = [8640, 3200], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<8640x3200xf16> | |
| %11 = tensor.empty(%8) : tensor<?x8640x3200xf16> | |
| %12 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%10 : tensor<8640x3200xf16>) outs(%11 : tensor<?x8640x3200xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x8640x3200xf16> | |
| flow.dispatch.tensor.store %12, %9, offsets = [0, 0, 0], sizes = [%8, 8640, 3200], strides = [1, 1, 1] : tensor<?x8640x3200xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%8} | |
| return | |
| } | |
| } | |
| } | |
| } | |
| // -----// IR Dump After TileAndDistributeToWorkgroups (iree-codegen-tile-and-distribute-to-workgroups) //----- // | |
| hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>], translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %c135 = arith.constant 135 : index | |
| %c50 = arith.constant 50 : index | |
| %0 = affine.apply affine_map<()[s0] -> (s0 ceildiv 64)>()[%arg1] | |
| hal.return %c50, %c135, %0 : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c64 = arith.constant 64 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [%c64, %c64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<?x?xf16> | |
| %17 = tensor.empty(%11) : tensor<?x64x64xf16> | |
| %cast = tensor.cast %16 : tensor<?x?xf16> to tensor<64x64xf16> | |
| %18 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%cast : tensor<64x64xf16>) outs(%17 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| %cast_0 = tensor.cast %18 : tensor<?x64x64xf16> to tensor<?x?x?xf16> | |
| %19 = arith.extui %0 : i32 to i64 | |
| %20 = arith.extui %1 : i32 to i64 | |
| %21 = arith.shli %20, %c32_i64 : i64 | |
| %22 = arith.ori %19, %21 : i64 | |
| %23 = arith.index_castui %22 : i64 to index | |
| flow.dispatch.tensor.store %cast_0, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, %c64, %c64], strides = [1, 1, 1] : tensor<?x?x?xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%23} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After ConvertToDestinationPassingStyle (iree-codegen-convert-to-destination-passing-style) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c64 = arith.constant 64 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = arith.extui %0 : i32 to i64 | |
| %17 = arith.extui %1 : i32 to i64 | |
| %18 = arith.shli %17, %c32_i64 : i64 | |
| %19 = arith.ori %16, %18 : i64 | |
| %20 = arith.index_castui %19 : i64 to index | |
| %21 = arith.extui %0 : i32 to i64 | |
| %22 = arith.extui %1 : i32 to i64 | |
| %23 = arith.shli %22, %c32_i64 : i64 | |
| %24 = arith.ori %21, %23 : i64 | |
| %25 = arith.index_castui %24 : i64 to index | |
| %26 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%25] | |
| %27 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%26, %c64, %c64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%20} -> tensor<?x?x?xf16> | |
| %cast = tensor.cast %27 : tensor<?x?x?xf16> to tensor<?x64x64xf16> | |
| %28 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [%c64, %c64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<?x?xf16> | |
| %cast_0 = tensor.cast %28 : tensor<?x?xf16> to tensor<64x64xf16> | |
| %29 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%cast_0 : tensor<64x64xf16>) outs(%cast : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| %cast_1 = tensor.cast %29 : tensor<?x64x64xf16> to tensor<?x?x?xf16> | |
| %30 = arith.extui %0 : i32 to i64 | |
| %31 = arith.extui %1 : i32 to i64 | |
| %32 = arith.shli %31, %c32_i64 : i64 | |
| %33 = arith.ori %30, %32 : i64 | |
| %34 = arith.index_castui %33 : i64 to index | |
| flow.dispatch.tensor.store %cast_1, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, %c64, %c64], strides = [1, 1, 1] : tensor<?x?x?xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%34} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After FoldAffineMinInDistributedLoops (iree-codegen-fold-affinemin-in-distributed-loops) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c64 = arith.constant 64 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = arith.extui %0 : i32 to i64 | |
| %17 = arith.extui %1 : i32 to i64 | |
| %18 = arith.shli %17, %c32_i64 : i64 | |
| %19 = arith.ori %16, %18 : i64 | |
| %20 = arith.index_castui %19 : i64 to index | |
| %21 = arith.extui %0 : i32 to i64 | |
| %22 = arith.extui %1 : i32 to i64 | |
| %23 = arith.shli %22, %c32_i64 : i64 | |
| %24 = arith.ori %21, %23 : i64 | |
| %25 = arith.index_castui %24 : i64 to index | |
| %26 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%25] | |
| %27 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%26, %c64, %c64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%20} -> tensor<?x?x?xf16> | |
| %cast = tensor.cast %27 : tensor<?x?x?xf16> to tensor<?x64x64xf16> | |
| %28 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [%c64, %c64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<?x?xf16> | |
| %cast_0 = tensor.cast %28 : tensor<?x?xf16> to tensor<64x64xf16> | |
| %29 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%cast_0 : tensor<64x64xf16>) outs(%cast : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| %cast_1 = tensor.cast %29 : tensor<?x64x64xf16> to tensor<?x?x?xf16> | |
| %30 = arith.extui %0 : i32 to i64 | |
| %31 = arith.extui %1 : i32 to i64 | |
| %32 = arith.shli %31, %c32_i64 : i64 | |
| %33 = arith.ori %30, %32 : i64 | |
| %34 = arith.index_castui %33 : i64 to index | |
| flow.dispatch.tensor.store %cast_1, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, %c64, %c64], strides = [1, 1, 1] : tensor<?x?x?xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%34} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = arith.extui %0 : i32 to i64 | |
| %17 = arith.extui %1 : i32 to i64 | |
| %18 = arith.shli %17, %c32_i64 : i64 | |
| %19 = arith.ori %16, %18 : i64 | |
| %20 = arith.index_castui %19 : i64 to index | |
| %21 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%20] | |
| %22 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%21, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %23 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %24 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%23 : tensor<64x64xf16>) outs(%22 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| flow.dispatch.tensor.store %24, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%17 : tensor<64x64xf16>) outs(%16 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After FuseTensorPadWithConsumer (iree-codegen-fuse-tensor-pad-with-consumer) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%17 : tensor<64x64xf16>) outs(%16 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After ConcretizePadResultShape (iree-codegen-concretize-pad-result-shape) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%17 : tensor<64x64xf16>) outs(%16 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After TileAndDecomposeWinogradTransform (iree-linalg-ext-tile-and-decompose-winograd) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%17 : tensor<64x64xf16>) outs(%16 : tensor<?x64x64xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<?x64x64xf16> | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUTileAndFuse (iree-llvmcpu-tile-and-fuse) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After FuseTensorPadWithConsumer (iree-codegen-fuse-tensor-pad-with-consumer) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After ConcretizePadResultShape (iree-codegen-concretize-pad-result-shape) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUSplitReduction (iree-llvmcpu-split-reduction) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUTile (iree-llvmcpu-tile) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUTileAndFuse (iree-llvmcpu-tile-and-fuse) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After FuseTensorPadWithConsumer (iree-codegen-fuse-tensor-pad-with-consumer) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After ConcretizePadResultShape (iree-codegen-concretize-pad-result-shape) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After TensorToVectorVectorizePad (iree-codegen-vectorize-tensor-pad) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%extracted_slice : tensor<16x16xf16>) outs(%extracted_slice_0 : tensor<1x16x16xf16>) attrs = {lowering_config = #iree_codegen.lowering_config<tile_sizes = [[64, 64, 64], [1, 16, 16], [0, 0, 0], [0, 0, 0]]>} { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } -> tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %21 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After GenericVectorization (iree-codegen-generic-vectorization) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %11 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| scf.for %arg1 = %12 to %c8640 step %13 { | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %15 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg2 = %14 to %c3200 step %15 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %11 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %extracted_slice = tensor.extract_slice %17[%arg5, %arg7] [16, 16] [1, 1] : tensor<64x64xf16> to tensor<16x16xf16> | |
| %extracted_slice_0 = tensor.extract_slice %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<?x64x64xf16> to tensor<1x16x16xf16> | |
| %21 = vector.transfer_read %extracted_slice[%c0, %c0], %cst {in_bounds = [true, true]} : tensor<16x16xf16>, vector<16x16xf16> | |
| %22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16> | |
| %23 = vector.transfer_write %22, %extracted_slice_0[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<1x16x16xf16> | |
| %inserted_slice = tensor.insert_slice %23 into %arg8[%arg3, %arg5, %arg7] [1, 16, 16] [1, 1, 1] : tensor<1x16x16xf16> into tensor<?x64x64xf16> | |
| scf.yield %inserted_slice : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%11, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After OptimizeTensorInsertExtractSlices (iree-codegen-optimize-tensor-insert-extract-slices) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %21 = vector.transfer_read %17[%arg5, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16> | |
| %22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16> | |
| %23 = vector.transfer_write %22, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<?x64x64xf16> | |
| scf.yield %23 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %21 = vector.transfer_read %17[%arg5, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16> | |
| %22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16> | |
| %23 = vector.transfer_write %22, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<?x64x64xf16> | |
| scf.yield %23 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %21 = vector.transfer_read %17[%arg5, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16> | |
| %22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16> | |
| %23 = vector.transfer_write %22, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<?x64x64xf16> | |
| scf.yield %23 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After EliminateEmptyTensors (iree-eliminate-empty-tensors) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %21 = vector.transfer_read %17[%arg5, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16> | |
| %22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16> | |
| %23 = vector.transfer_write %22, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<?x64x64xf16> | |
| scf.yield %23 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After EmptyTensorToAllocTensor (empty-tensor-to-alloc-tensor) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %16 = flow.dispatch.tensor.load %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} -> tensor<?x64x64xf16> | |
| %17 = flow.dispatch.tensor.load %7, offsets = [%arg1, %arg2], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<8640x3200xf16>> -> tensor<64x64xf16> | |
| %18 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %16) -> (tensor<?x64x64xf16>) { | |
| %19 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (tensor<?x64x64xf16>) { | |
| %20 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (tensor<?x64x64xf16>) { | |
| %21 = vector.transfer_read %17[%arg5, %arg7], %cst {in_bounds = [true, true]} : tensor<64x64xf16>, vector<16x16xf16> | |
| %22 = vector.broadcast %21 : vector<16x16xf16> to vector<1x16x16xf16> | |
| %23 = vector.transfer_write %22, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, tensor<?x64x64xf16> | |
| scf.yield %23 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %20 : tensor<?x64x64xf16> | |
| } | |
| scf.yield %19 : tensor<?x64x64xf16> | |
| } | |
| flow.dispatch.tensor.store %18, %8, offsets = [%arg0, %arg1, %arg2], sizes = [%15, 64, 64], strides = [1, 1, 1] : tensor<?x64x64xf16> -> !flow.dispatch.tensor<writeonly:tensor<?x8640x3200xf16>>{%6} | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After IREEComprehensiveBufferize (iree-codegen-iree-comprehensive-bufferize) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %16 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %subview) -> (memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| %17 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| %18 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| %19 = vector.transfer_read %subview_0[%arg5, %arg7], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %20 = vector.broadcast %19 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %20, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.yield %arg8 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| scf.yield %18 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| scf.yield %17 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| %subview_1 = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%16 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview_1 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After ResolveShapedTypeResultDims (resolve-shaped-type-result-dims) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %16 = scf.for %arg3 = %c0 to %15 step %c1 iter_args(%arg4 = %subview) -> (memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| %17 = scf.for %arg5 = %c0 to %c64 step %c16 iter_args(%arg6 = %arg4) -> (memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| %18 = scf.for %arg7 = %c0 to %c64 step %c16 iter_args(%arg8 = %arg6) -> (memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| %19 = vector.transfer_read %subview_0[%arg5, %arg7], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %20 = vector.broadcast %19 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %20, %arg8[%arg3, %arg5, %arg7] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.yield %arg8 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| scf.yield %18 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| scf.yield %17 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| %subview_1 = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%16 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview_1 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %17 = vector.broadcast %16 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %17, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| %subview_1 = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%subview : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview_1 : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %17 = vector.broadcast %16 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %17, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| linalg.generic {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1, d2)>], iterator_types = ["parallel", "parallel", "parallel"]} ins(%subview : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) outs(%subview : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>) { | |
| ^bb0(%in: f16, %out: f16): | |
| linalg.yield %in : f16 | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %17 = vector.broadcast %16 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %17, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After CleanupBufferAllocView (iree-codegen-cleanup-buffer-alloc-view) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %17 = vector.broadcast %16 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %17, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After RemoveSingleIterationLoop (iree-codegen-remove-single-iteration-loop) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| %17 = vector.broadcast %16 : vector<16x16xf16> to vector<1x16x16xf16> | |
| vector.transfer_write %17, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true, true]} : vector<1x16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUDropVectorUnitDims (iree-llvmcpu-drop-vector-unit-dims) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| vector.transfer_write %16, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true]} : vector<16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUVirtualVectorLowering (iree-llvmcpu-virtual-vector-lowering) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| vector.transfer_write %16, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true]} : vector<16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %cst = arith.constant 0.000000e+00 : f16 | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.transfer_read %subview_0[%arg4, %arg5], %cst {in_bounds = [true, true]} : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16x16xf16> | |
| vector.transfer_write %16, %subview[%arg3, %arg4, %arg5] {in_bounds = [true, true]} : vector<16x16xf16>, memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUVectorTransferLowering (iree-llvmcpu-vector-transfer-lowering) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| Has 16x16 transpose: 0 | |
| // -----// IR Dump After LLVMCPUVectorTransposeLowering (iree-llvmcpu-vector-transpose-lowering) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPUVectorShapeCastLowering (iree-llvmcpu-vector-shape-cast-lowering) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After LLVMCPULowerExecutableTarget (iree-llvmcpu-lower-executable-target) //----- // | |
| hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu = "znver4", cpu_features = "+prfchw,-cldemote,+avx,+aes,+sahf,+pclmul,-xop,+crc32,+xsaves,-avx512fp16,-usermsr,-sm4,-egpr,+sse4.1,+avx512ifma,+xsave,-avx512pf,+sse4.2,-tsxldtrk,-ptwrite,-widekl,-sm3,+invpcid,+64bit,+xsavec,-avx10.1-512,+avx512vpopcntdq,+cmov,-avx512vp2intersect,+avx512cd,+movbe,-avxvnniint8,-avx512er,-ccmp,-amx-int8,-kl,-avx10.1-256,-sha512,-avxvnni,-rtm,+adx,+avx2,-hreset,-movdiri,-serialize,+vpclmulqdq,+avx512vl,-uintr,-cf,+clflushopt,-raoint,-cmpccxadd,+bmi,-amx-tile,+sse,+gfni,-avxvnniint16,-amx-fp16,-ndd,+xsaveopt,+rdrnd,+avx512f,-amx-bf16,+avx512bf16,+avx512vnni,-push2pop2,+cx8,+avx512bw,+sse3,+pku,+fsgsbase,+clzero,+mwaitx,-lwp,+lzcnt,+sha,-movdir64b,-ppx,+wbnoinvd,-enqcmd,-prefetchwt1,-avxneconvert,-tbm,-pconfig,-amx-complex,+ssse3,+cx16,+bmi2,+fma,+popcnt,-avxifma,+f16c,+avx512bitalg,+rdpru,+clwb,+mmx,+sse2,+rdseed,+avx512vbmi2,-prefetchi,+rdpid,-fma4,+avx512vbmi,+shstk,+vaes,-waitpkg,-sgx,+fxsr,+avx512dq,+sse4a", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 64 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}>) { | |
| hal.executable.export public @broadcast_dispatch_0_generic_Dx8640x3200_f16 ordinal(0) layout(#hal.pipeline.layout<push_constants = 2, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) attributes {hal.interface.bindings = [#hal.interface.binding<0, 0>, #hal.interface.binding<0, 1>], translation_info = #iree_codegen.translation_info<CPUDoubleTilingExpert>} { | |
| ^bb0(%arg0: !hal.device, %arg1: index): | |
| %c135 = arith.constant 135 : index | |
| %c50 = arith.constant 50 : index | |
| %0 = affine.apply affine_map<()[s0] -> (s0 ceildiv 64)>()[%arg1] | |
| hal.return %c50, %c135, %0 : index, index, index | |
| } | |
| builtin.module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16, #hal.descriptor_type<storage_buffer>> to memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>, #hal.descriptor_type<storage_buffer>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| } | |
| // -----// IR Dump After EraseHALDescriptorTypeFromMemRef (iree-codegen-erase-hal-descriptor-type-from-memref) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After LowerUKernelOpsToCalls (iree-codegen-lower-ukernel-ops-to-calls) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After LinalgExtToLoops (iree-linalg-ext-to-loops) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After MemrefCopyToLinalgPass (iree-codegen-memrefcopy-to-linalg) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After ConvertLinalgToLoopsPass (convert-linalg-to-loops) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After ConvertBf16ArithToF32 (iree-convert-bf16-arith-to-f32) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After ConvertBf16ToUInt16Buffers (iree-convert-bf16-to-uint16-buffers) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After Canonicalizer (canonicalize) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %47 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| vector.store %18, %subview[%arg3, %47, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %48 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| vector.store %20, %subview[%arg3, %48, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %49 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| vector.store %22, %subview[%arg3, %49, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %50 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| vector.store %24, %subview[%arg3, %50, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %51 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| vector.store %26, %subview[%arg3, %51, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %52 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| vector.store %28, %subview[%arg3, %52, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %53 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| vector.store %30, %subview[%arg3, %53, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %54 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| vector.store %32, %subview[%arg3, %54, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %55 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| vector.store %34, %subview[%arg3, %55, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %56 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| vector.store %36, %subview[%arg3, %56, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %57 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| vector.store %38, %subview[%arg3, %57, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %58 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| vector.store %40, %subview[%arg3, %58, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %59 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| vector.store %42, %subview[%arg3, %59, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %60 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| vector.store %44, %subview[%arg3, %60, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| %61 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| vector.store %46, %subview[%arg3, %61, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After CSE (cse) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %18, %subview[%arg3, %17, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %20, %subview[%arg3, %19, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %22, %subview[%arg3, %21, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %24, %subview[%arg3, %23, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %26, %subview[%arg3, %25, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %28, %subview[%arg3, %27, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %30, %subview[%arg3, %29, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %32, %subview[%arg3, %31, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %34, %subview[%arg3, %33, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %36, %subview[%arg3, %35, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %38, %subview[%arg3, %37, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %40, %subview[%arg3, %39, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %42, %subview[%arg3, %41, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %44, %subview[%arg3, %43, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %46, %subview[%arg3, %45, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| // -----// IR Dump After ArithBufferizePass (arith-bufferize) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %18, %subview[%arg3, %17, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %20, %subview[%arg3, %19, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %22, %subview[%arg3, %21, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %24, %subview[%arg3, %23, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %26, %subview[%arg3, %25, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %28, %subview[%arg3, %27, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %30, %subview[%arg3, %29, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %32, %subview[%arg3, %31, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %34, %subview[%arg3, %33, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %36, %subview[%arg3, %35, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %38, %subview[%arg3, %37, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %40, %subview[%arg3, %39, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %42, %subview[%arg3, %41, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %44, %subview[%arg3, %43, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %46, %subview[%arg3, %45, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After FoldTensorExtractOp (iree-codegen-fold-tensor-extract-op) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %18, %subview[%arg3, %17, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %20, %subview[%arg3, %19, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %22, %subview[%arg3, %21, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %24, %subview[%arg3, %23, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %26, %subview[%arg3, %25, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %28, %subview[%arg3, %27, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %30, %subview[%arg3, %29, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %32, %subview[%arg3, %31, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %34, %subview[%arg3, %33, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %36, %subview[%arg3, %35, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %38, %subview[%arg3, %37, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %40, %subview[%arg3, %39, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %42, %subview[%arg3, %41, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %44, %subview[%arg3, %43, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %46, %subview[%arg3, %45, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After ConvertComplexToStandard (convert-complex-to-standard) //----- // | |
| module { | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c0) : memref<?x8640x3200xf16>{%6} | |
| memref.assume_alignment %8, 64 : memref<?x8640x3200xf16> | |
| %workgroup_id_x = hal.interface.workgroup.id[0] : index | |
| %workgroup_count_x = hal.interface.workgroup.count[0] : index | |
| %workgroup_id_y = hal.interface.workgroup.id[1] : index | |
| %workgroup_count_y = hal.interface.workgroup.count[1] : index | |
| %workgroup_id_z = hal.interface.workgroup.id[2] : index | |
| %workgroup_count_z = hal.interface.workgroup.count[2] : index | |
| %9 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_z] | |
| %10 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_z] | |
| %11 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_y] | |
| %12 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_y] | |
| %13 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_id_x] | |
| %14 = affine.apply affine_map<()[s0] -> (s0 * 64)>()[%workgroup_count_x] | |
| scf.for %arg0 = %9 to %6 step %10 { | |
| %15 = affine.min affine_map<(d0)[s0] -> (-d0 + s0, 64)>(%arg0)[%6] | |
| scf.for %arg1 = %11 to %c8640 step %12 { | |
| scf.for %arg2 = %13 to %c3200 step %14 { | |
| %subview = memref.subview %8[%arg0, %arg1, %arg2] [%15, 64, 64] [1, 1, 1] : memref<?x8640x3200xf16> to memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>> | |
| %subview_0 = memref.subview %7[%arg1, %arg2] [64, 64] [1, 1] : memref<8640x3200xf16> to memref<64x64xf16, strided<[3200, 1], offset: ?>> | |
| scf.for %arg3 = %c0 to %15 step %c1 { | |
| scf.for %arg4 = %c0 to %c64 step %c16 { | |
| scf.for %arg5 = %c0 to %c64 step %c16 { | |
| %16 = vector.load %subview_0[%arg4, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %17 = affine.apply affine_map<(d0) -> (d0 + 1)>(%arg4) | |
| %18 = vector.load %subview_0[%17, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %19 = affine.apply affine_map<(d0) -> (d0 + 2)>(%arg4) | |
| %20 = vector.load %subview_0[%19, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %21 = affine.apply affine_map<(d0) -> (d0 + 3)>(%arg4) | |
| %22 = vector.load %subview_0[%21, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %23 = affine.apply affine_map<(d0) -> (d0 + 4)>(%arg4) | |
| %24 = vector.load %subview_0[%23, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %25 = affine.apply affine_map<(d0) -> (d0 + 5)>(%arg4) | |
| %26 = vector.load %subview_0[%25, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %27 = affine.apply affine_map<(d0) -> (d0 + 6)>(%arg4) | |
| %28 = vector.load %subview_0[%27, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %29 = affine.apply affine_map<(d0) -> (d0 + 7)>(%arg4) | |
| %30 = vector.load %subview_0[%29, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %31 = affine.apply affine_map<(d0) -> (d0 + 8)>(%arg4) | |
| %32 = vector.load %subview_0[%31, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %33 = affine.apply affine_map<(d0) -> (d0 + 9)>(%arg4) | |
| %34 = vector.load %subview_0[%33, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %35 = affine.apply affine_map<(d0) -> (d0 + 10)>(%arg4) | |
| %36 = vector.load %subview_0[%35, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %37 = affine.apply affine_map<(d0) -> (d0 + 11)>(%arg4) | |
| %38 = vector.load %subview_0[%37, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %39 = affine.apply affine_map<(d0) -> (d0 + 12)>(%arg4) | |
| %40 = vector.load %subview_0[%39, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %41 = affine.apply affine_map<(d0) -> (d0 + 13)>(%arg4) | |
| %42 = vector.load %subview_0[%41, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %43 = affine.apply affine_map<(d0) -> (d0 + 14)>(%arg4) | |
| %44 = vector.load %subview_0[%43, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| %45 = affine.apply affine_map<(d0) -> (d0 + 15)>(%arg4) | |
| %46 = vector.load %subview_0[%45, %arg5] : memref<64x64xf16, strided<[3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %16, %subview[%arg3, %arg4, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %18, %subview[%arg3, %17, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %20, %subview[%arg3, %19, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %22, %subview[%arg3, %21, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %24, %subview[%arg3, %23, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %26, %subview[%arg3, %25, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %28, %subview[%arg3, %27, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %30, %subview[%arg3, %29, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %32, %subview[%arg3, %31, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %34, %subview[%arg3, %33, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %36, %subview[%arg3, %35, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %38, %subview[%arg3, %37, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %40, %subview[%arg3, %39, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %42, %subview[%arg3, %41, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %44, %subview[%arg3, %43, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| vector.store %46, %subview[%arg3, %45, %arg5] : memref<?x64x64xf16, strided<[27648000, 3200, 1], offset: ?>>, vector<16xf16> | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| return | |
| } | |
| } | |
| // -----// IR Dump After PolynomialApproximationPass (iree-codegen-polynomial-approximation) //----- // | |
| func.func @broadcast_dispatch_0_generic_Dx8640x3200_f16() { | |
| %c16 = arith.constant 16 : index | |
| %c64 = arith.constant 64 : index | |
| %c1 = arith.constant 1 : index | |
| %c3200 = arith.constant 3200 : index | |
| %c8640 = arith.constant 8640 : index | |
| %c0 = arith.constant 0 : index | |
| %c32_i64 = arith.constant 32 : i64 | |
| %0 = hal.interface.constant.load[0] : i32 | |
| %1 = hal.interface.constant.load[1] : i32 | |
| %2 = arith.extui %0 : i32 to i64 | |
| %3 = arith.extui %1 : i32 to i64 | |
| %4 = arith.shli %3, %c32_i64 : i64 | |
| %5 = arith.ori %2, %4 : i64 | |
| %6 = arith.index_castui %5 : i64 to index | |
| %7 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c0) flags(ReadOnly) : memref<8640x3200xf16> | |
| memref.assume_alignment %7, 64 : memref<8640x3200xf16> | |
| %8 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(% |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment