Created
October 3, 2024 16:09
-
-
Save pashu123/24b9ec7287c4c3f523d2dd2996bafca0 to your computer and use it in GitHub Desktop.
This file has been truncated, but you can view the full file.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// -----// IR Dump After AutoInputConversionPipelinePass (iree-auto-input-conversion) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After IREEImportPublicPass (iree-import-public) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After ImportMLProgramPass (iree-import-ml-program) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After SanitizeModuleNamesPass (iree-sanitize-module-names) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After ConvertMeshToFlowPass (iree-convert-mesh-to-flow) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After DemoteF64ToF32Pass (iree-input-conversion-demote-f64-to-f32) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After mlir::iree_compiler::IREE::ABI::ConvertStreamableOpsPass (iree-abi-convert-streamable-ops) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After mlir::iree_compiler::IREE::ABI::WrapEntryPointsPass (iree-abi-wrap-entry-points) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = util.call @_matmul(%0, %1) : (tensor<64x64xf32>, tensor<64x64xf32>) -> tensor<64x64xf32> | |
%3 = hal.tensor.export %2 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
util.func private @_matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @_matmul) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = util.call @_matmul(%0, %1) : (tensor<64x64xf32>, tensor<64x64xf32>) -> tensor<64x64xf32> | |
%3 = hal.tensor.export %2 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
util.func private @_matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = util.call @_matmul(%0, %1) : (tensor<64x64xf32>, tensor<64x64xf32>) -> tensor<64x64xf32> | |
%3 = hal.tensor.export %2 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
util.func private @_matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = tensor.empty() : tensor<64x64xf32> | |
%1 = linalg.fill ins(%cst : f32) outs(%0 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%2 = linalg.matmul ins(%arg0, %arg1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
util.return %2 : tensor<64x64xf32> | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
util.func private @_matmul(%arg0: tensor<64x64xf32>, %arg1: tensor<64x64xf32>) -> tensor<64x64xf32> | |
} | |
// -----// IR Dump After Inliner (inline) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SymbolDCE (symbol-dce) ('builtin.module' operation) //----- // | |
module { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After AssignLegacyTargetDevicesPass (iree-hal-assign-legacy-target-devices) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {hal.device.targets = [#device_target_local]} { | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After MaterializeTargetDevicesPass (iree-hal-materialize-target-devices) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ResolveDevicePromisesPass (iree-hal-resolve-device-promises) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ResolveDeviceAliasesPass (iree-hal-resolve-device-aliases) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyDevicesPass (iree-hal-verify-devices) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After LinalgQuantizedConvToConvPass (iree-global-opt-quantized-conv-to-conv) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After LinalgQuantizedMatmulToMatmulPass (iree-global-opt-quantized-matmul-to-matmul) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After RemoveZeroExtentTensorsPass (iree-global-opt-remove-zero-extent-tensors) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After DetachElementwiseFromNamedOpsPass (iree-global-opt-detach-elementwise-from-named-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After LinalgNamedOpConversionPass (linalg-named-op-conversion) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Convert1X1FilterConv2DToMatmulPass (iree-global-opt-convert-1x1-filter-conv2d-to-matmul) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EraseUnusedLinalgOperandsPass (iree-global-opt-erase-unused-linalg-operands) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ExpandTensorShapesPass (iree-global-opt-expand-tensor-shapes) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ConvertElementwiseToLinalgPass (convert-elementwise-to-linalg) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After RaiseSpecialOpsPass (iree-global-opt-raise-special-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After DecomposeConcatPass (iree-global-opt-decompose-concat) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After GeneralizeLinalgNamedOpsPass (iree-global-opt-generalize-linalg-named-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldUnitExtentDimsPass (iree-dispatch-creation-fold-unit-extent-dims) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After DemoteContractionInputsToBF16Pass (iree-global-opt-demote-contraction-inputs-to-bf16) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<64x64xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%4 = linalg.matmul ins(%0, %1 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%3 : tensor<64x64xf32>) -> tensor<64x64xf32> | |
%5 = hal.tensor.export %4 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %5 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SetEncodingPass (iree-dispatch-creation-set-encoding) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#map = affine_map<(d0, d1, d2) -> (d0, d2)> | |
#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> | |
#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = iree_encoding.set_encoding %0 : tensor<64x64xf32> -> tensor<64x64xf32, #iree_encoding.encoding<operand_index = 0 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>> | |
%3 = iree_encoding.set_encoding %1 : tensor<64x64xf32> -> tensor<64x64xf32, #iree_encoding.encoding<operand_index = 1 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>> | |
%4 = tensor.empty() : tensor<64x64xf32, #iree_encoding.encoding<operand_index = 2 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<64x64xf32, #iree_encoding.encoding<operand_index = 2 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>>) -> tensor<64x64xf32, #iree_encoding.encoding<operand_index = 2 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>> | |
%6 = linalg.matmul ins(%2, %3 : tensor<64x64xf32, #iree_encoding.encoding<operand_index = 0 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>>, tensor<64x64xf32, #iree_encoding.encoding<operand_index = 1 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>>) outs(%5 : tensor<64x64xf32, #iree_encoding.encoding<operand_index = 2 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>>) -> tensor<64x64xf32, #iree_encoding.encoding<operand_index = 2 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>> | |
%7 = iree_encoding.unset_encoding %6 : tensor<64x64xf32, #iree_encoding.encoding<operand_index = 2 : index, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], round_dims_to = array<i64: 32, 32, 32>>> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %7 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CPUMaterializeHostEncodingPass (iree-codegen-cpu-materialize-host-encoding) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 padding_value(%cst : f32) outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 padding_value(%cst : f32) outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After MaterializeHomogeneousEncodingsPass (iree-global-opt-materialize-homogeneous-encodings) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 padding_value(%cst : f32) outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 padding_value(%cst : f32) outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyPackUnpackPass (iree-global-opt-simplify-pack-unpack) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After DataLayoutPropagationPass (iree-global-opt-data-layout-propagation) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After GeneralizeLinalgNamedOpsPass (iree-global-opt-generalize-linalg-named-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After GlobalLoopInvariantCodeMotionPass (iree-global-opt-loop-invariant-code-motion) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After HoistIntoGlobals (iree-util-hoist-into-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After JitGlobalsPass (iree-consteval-jit-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After RaiseSpecialOpsPass (iree-global-opt-raise-special-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After InjectTensorTracingPass (iree-flow-inject-tensor-tracing) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After TensorPadToTensorInsertSlicePass (iree-dispatch-creation-tensor-pad-to-tensor-insert-slice) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FixedPointIterator (iree-util-fixed-point-iterator) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FusionPreprocessingPass (iree-dispatch-creation-fusion-preprocessing) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ElementwiseOpFusionPass (iree-dispatch-creation-elementwise-op-fusion) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After BubbleUpExpandShapesPass (iree-dispatch-creation-bubble-up-expand-shapes) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After BubbleUpExtractSlicesPass (iree-dispatch-creation-bubble-up-extract-slices) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ElementwiseOpFusionPass (iree-dispatch-creation-elementwise-op-fusion) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SinkReshapesPass (iree-dispatch-creation-sink-reshapes) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseMultiUseElementwiseProducerPass (iree-dispatch-creation-fuse-multi-use-elementwise-producer) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SplitReductionPass (iree-dispatch-creation-split-reduction-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After TransposeGenericOpsPass (iree-dispatch-creation-transpose-generic-ops) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FormScalarDispatchesPass (iree-dispatch-creation-form-scalar-dispatches) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack_0 = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
%4 = tensor.empty() : tensor<8x16x8x4xf32> | |
%5 = linalg.fill ins(%cst : f32) outs(%4 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%6 = linalg.mmt4d ins(%pack, %pack_0 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %6 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %7 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
%8 = hal.tensor.export %unpack "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %8 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FormDispatchRegionsPass (iree-dispatch-creation-form-dispatch-regions) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%3 = flow.dispatch.region -> (tensor<8x64x8x1xf32>) { | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %2 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.return %pack : tensor<8x64x8x1xf32> | |
} | |
%4 = tensor.empty() : tensor<16x64x4x1xf32> | |
%5 = flow.dispatch.region -> (tensor<16x64x4x1xf32>) { | |
%pack = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %4 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.return %pack : tensor<16x64x4x1xf32> | |
} | |
%6 = tensor.empty() : tensor<8x16x8x4xf32> | |
%7 = linalg.fill ins(%cst : f32) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%8 = flow.dispatch.region -> (tensor<8x16x8x4xf32>) { | |
%12 = linalg.mmt4d ins(%3, %5 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%7 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.return %12 : tensor<8x16x8x4xf32> | |
} | |
%9 = tensor.empty() : tensor<64x64xf32> | |
%10 = flow.dispatch.region -> (tensor<64x64xf32>) { | |
%unpack = tensor.unpack %8 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %9 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.return %unpack : tensor<64x64xf32> | |
} | |
%11 = hal.tensor.export %10 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %11 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CloneProducersIntoDispatchRegionsPass (iree-dispatch-creation-clone-producers-into-dispatch-regions) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%3 = flow.dispatch.region -> (tensor<8x64x8x1xf32>) { | |
%12 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %12 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.return %pack : tensor<8x64x8x1xf32> | |
} | |
%4 = tensor.empty() : tensor<16x64x4x1xf32> | |
%5 = flow.dispatch.region -> (tensor<16x64x4x1xf32>) { | |
%12 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %12 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.return %pack : tensor<16x64x4x1xf32> | |
} | |
%6 = tensor.empty() : tensor<8x16x8x4xf32> | |
%7 = linalg.fill ins(%cst : f32) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%8 = flow.dispatch.region -> (tensor<8x16x8x4xf32>) { | |
%12 = tensor.empty() : tensor<8x16x8x4xf32> | |
%cst_0 = arith.constant 0.000000e+00 : f32 | |
%13 = linalg.fill ins(%cst_0 : f32) outs(%12 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%14 = linalg.mmt4d ins(%3, %5 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%13 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.return %14 : tensor<8x16x8x4xf32> | |
} | |
%9 = tensor.empty() : tensor<64x64xf32> | |
%10 = flow.dispatch.region -> (tensor<64x64xf32>) { | |
%12 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %8 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %12 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.return %unpack : tensor<64x64xf32> | |
} | |
%11 = hal.tensor.export %10 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %11 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CollapseDimensionsPass (iree-dispatch-creation-collapse-dimensions) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%3 = flow.dispatch.region -> (tensor<8x64x8x1xf32>) { | |
%12 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %12 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.return %pack : tensor<8x64x8x1xf32> | |
} | |
%4 = tensor.empty() : tensor<16x64x4x1xf32> | |
%5 = flow.dispatch.region -> (tensor<16x64x4x1xf32>) { | |
%12 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %1 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %12 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.return %pack : tensor<16x64x4x1xf32> | |
} | |
%6 = tensor.empty() : tensor<8x16x8x4xf32> | |
%7 = linalg.fill ins(%cst : f32) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%8 = flow.dispatch.region -> (tensor<8x16x8x4xf32>) { | |
%12 = tensor.empty() : tensor<8x16x8x4xf32> | |
%cst_0 = arith.constant 0.000000e+00 : f32 | |
%13 = linalg.fill ins(%cst_0 : f32) outs(%12 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%14 = linalg.mmt4d ins(%3, %5 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%13 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.return %14 : tensor<8x16x8x4xf32> | |
} | |
%9 = tensor.empty() : tensor<64x64xf32> | |
%10 = flow.dispatch.region -> (tensor<64x64xf32>) { | |
%12 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %8 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %12 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.return %unpack : tensor<64x64xf32> | |
} | |
%11 = hal.tensor.export %10 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %11 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ConvertDispatchRegionsToWorkgroupsPass (iree-dispatch-creation-convert-dispatch-regions-to-workgroups) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = tensor.empty() : tensor<8x64x8x1xf32> | |
%3 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%12 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%13 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %12 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %13 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} | |
%4 = tensor.empty() : tensor<16x64x4x1xf32> | |
%5 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%12 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%13 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %12 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %13 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} | |
%6 = tensor.empty() : tensor<8x16x8x4xf32> | |
%7 = linalg.fill ins(%cst : f32) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%8 = flow.dispatch.workgroups(%3, %5) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%12 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%13 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%14 = tensor.empty() : tensor<8x16x8x4xf32> | |
%cst_0 = arith.constant 0.000000e+00 : f32 | |
%15 = linalg.fill ins(%cst_0 : f32) outs(%14 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%16 = linalg.mmt4d ins(%12, %13 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%15 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %16, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} | |
%9 = tensor.empty() : tensor<64x64xf32> | |
%10 = flow.dispatch.workgroups(%8) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%12 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%13 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %12 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %13 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} | |
%11 = hal.tensor.export %10 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %11 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ConvertTensorToFlowPass (iree-dispatch-creation-convert-tensor-to-flow) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After MaterializeDefaultWorkgroupCountRegionPass (iree-dispatch-creation-materialize-default-workgroup-count-region) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyInputLegalityPass (iree-verify-input-legality) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CaptureDynamicDimsPass (iree-flow-capture-dynamic-dims) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After InitializeEmptyTensorsPass (iree-flow-initialize-empty-tensors) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After OutlineDispatchExternsPass (iree-flow-outline-dispatch-externs) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch.workgroups(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %8 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%3 = flow.dispatch.workgroups(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%8 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %7 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %8 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%4 = flow.dispatch.workgroups(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg3: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg4: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%8 = flow.dispatch.tensor.load %arg3, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%9 = tensor.empty() : tensor<8x16x8x4xf32> | |
%10 = linalg.fill ins(%cst : f32) outs(%9 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%11 = linalg.mmt4d ins(%7, %8 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%10 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %11, %arg4, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%5 = flow.dispatch.workgroups(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> = | |
(%arg2: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg3: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%7 = flow.dispatch.tensor.load %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%8 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %7 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %8 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg3, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
flow.return | |
} count() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After OutlineDispatchRegionsPass (iree-flow-outline-dispatch-regions) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After AnnotateDispatchesPass (iree-flow-annotate-dispatches) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After StripDebugOps (iree-util-strip-debug-ops) ('flow.executable' operation: @matmul_dispatch_0) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After StripDebugOps (iree-util-strip-debug-ops) ('flow.executable' operation: @matmul_dispatch_1) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After StripDebugOps (iree-util-strip-debug-ops) ('flow.executable' operation: @matmul_dispatch_2) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After StripDebugOps (iree-util-strip-debug-ops) ('flow.executable' operation: @matmul_dispatch_3) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After DeduplicateExecutablesPass (iree-flow-deduplicate-executables) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After InjectTensorTracingPass (iree-flow-inject-tensor-tracing) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CleanupTensorShapesPass (iree-flow-cleanup-tensor-shapes) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After OutlineConstantsPass (iree-flow-outline-constants) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CanonicalizerPass (iree-flow-canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {iree.fixedpoint.iteration = 0 : index, stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FixedPointIterator (iree-util-fixed-point-iterator) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SymbolDCE (symbol-dce) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyInputPass (iree-stream-verify-input) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
flow.executable private @matmul_dispatch_0 { | |
flow.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %1 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_1 { | |
flow.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<64x64xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%1 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %0 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %1 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_2 { | |
flow.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>>, %arg1: !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>>, %arg2: !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>>) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%1 = flow.dispatch.tensor.load %arg1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%2 = tensor.empty() : tensor<8x16x8x4xf32> | |
%3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%4 = linalg.mmt4d ins(%0, %1 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%3 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %4, %arg2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
flow.executable private @matmul_dispatch_3 { | |
flow.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
flow.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>>, %arg1: !flow.dispatch.tensor<writeonly:tensor<64x64xf32>>) { | |
%0 = flow.dispatch.tensor.load %arg0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%1 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %0 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %1 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %arg1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%0 = hal.tensor.import %arg0 "input0" : !hal.buffer_view -> tensor<64x64xf32> | |
%1 = hal.tensor.import %arg1 "input1" : !hal.buffer_view -> tensor<64x64xf32> | |
%2 = flow.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0) : (tensor<64x64xf32>) -> tensor<8x64x8x1xf32> | |
%3 = flow.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1) : (tensor<64x64xf32>) -> tensor<16x64x4x1xf32> | |
%4 = flow.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2, %3) : (tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) -> tensor<8x16x8x4xf32> | |
%5 = flow.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4) : (tensor<8x16x8x4xf32>) -> tensor<64x64xf32> | |
%6 = hal.tensor.export %5 "output0" : tensor<64x64xf32> -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ConvertToStreamPass (iree-stream-conversion) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%cst = arith.constant 0.000000e+00 : f32 | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
%c64 = arith.constant 64 : index | |
%c64_0 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64_0]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_1 = hal.element_type<f32> : i32 | |
%dense_row_major_2 = hal.encoding_type<dense_row_major> : i32 | |
%c64_3 = arith.constant 64 : index | |
%c64_4 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64_3, %c64_4]) type(%element_type_f32_1) encoding(%dense_row_major_2) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%c0 = arith.constant 0 : index | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%c0_5 = arith.constant 0 : index | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0_5 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%c0_6 = arith.constant 0 : index | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0_6 to %6 for %6], %9[%c0_6 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%c0_7 = arith.constant 0 : index | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0_7 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyLoweringToTensorsPass (iree-stream-verify-lowering-to-tensors) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%cst = arith.constant 0.000000e+00 : f32 | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
%c64 = arith.constant 64 : index | |
%c64_0 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64_0]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_1 = hal.element_type<f32> : i32 | |
%dense_row_major_2 = hal.encoding_type<dense_row_major> : i32 | |
%c64_3 = arith.constant 64 : index | |
%c64_4 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64_3, %c64_4]) type(%element_type_f32_1) encoding(%dense_row_major_2) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%c0 = arith.constant 0 : index | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%c0_5 = arith.constant 0 : index | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0_5 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%c0_6 = arith.constant 0 : index | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0_6 to %6 for %6], %9[%c0_6 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%c0_7 = arith.constant 0 : index | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0_7 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('func.func' operation: @matmul_dispatch_0_pack_f32) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%cst = arith.constant 0.000000e+00 : f32 | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
%c64 = arith.constant 64 : index | |
%c64_0 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64_0]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_1 = hal.element_type<f32> : i32 | |
%dense_row_major_2 = hal.encoding_type<dense_row_major> : i32 | |
%c64_3 = arith.constant 64 : index | |
%c64_4 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64_3, %c64_4]) type(%element_type_f32_1) encoding(%dense_row_major_2) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%c0 = arith.constant 0 : index | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%c0_5 = arith.constant 0 : index | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0_5 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%c0_6 = arith.constant 0 : index | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0_6 to %6 for %6], %9[%c0_6 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%c0_7 = arith.constant 0 : index | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0_7 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('func.func' operation: @matmul_dispatch_1_pack_f32) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%cst = arith.constant 0.000000e+00 : f32 | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
%c64 = arith.constant 64 : index | |
%c64_0 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64_0]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_1 = hal.element_type<f32> : i32 | |
%dense_row_major_2 = hal.encoding_type<dense_row_major> : i32 | |
%c64_3 = arith.constant 64 : index | |
%c64_4 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64_3, %c64_4]) type(%element_type_f32_1) encoding(%dense_row_major_2) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%c0 = arith.constant 0 : index | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%c0_5 = arith.constant 0 : index | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0_5 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%c0_6 = arith.constant 0 : index | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0_6 to %6 for %6], %9[%c0_6 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%c0_7 = arith.constant 0 : index | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0_7 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('func.func' operation: @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
%c64 = arith.constant 64 : index | |
%c64_0 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64_0]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_1 = hal.element_type<f32> : i32 | |
%dense_row_major_2 = hal.encoding_type<dense_row_major> : i32 | |
%c64_3 = arith.constant 64 : index | |
%c64_4 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64_3, %c64_4]) type(%element_type_f32_1) encoding(%dense_row_major_2) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%c0 = arith.constant 0 : index | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%c0_5 = arith.constant 0 : index | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0_5 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%c0_6 = arith.constant 0 : index | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0_6 to %6 for %6], %9[%c0_6 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%c0_7 = arith.constant 0 : index | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0_7 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('func.func' operation: @matmul_dispatch_3_unpack_f32) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
%c64 = arith.constant 64 : index | |
%c64_0 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64_0]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_1 = hal.element_type<f32> : i32 | |
%dense_row_major_2 = hal.encoding_type<dense_row_major> : i32 | |
%c64_3 = arith.constant 64 : index | |
%c64_4 = arith.constant 64 : index | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64_3, %c64_4]) type(%element_type_f32_1) encoding(%dense_row_major_2) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%c0 = arith.constant 0 : index | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%c0_5 = arith.constant 0 : index | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0_5 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%c0_6 = arith.constant 0 : index | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0_6 to %6 for %6], %9[%c0_6 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%c0_7 = arith.constant 0 : index | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0_7 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_0 = hal.element_type<f32> : i32 | |
%dense_row_major_1 = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32_0) encoding(%dense_row_major_1) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0 to %6 for %6], %9[%c0 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Inliner (inline) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_0 = hal.element_type<f32> : i32 | |
%dense_row_major_1 = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32_0) encoding(%dense_row_major_1) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0 to %6 for %6], %9[%c0 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%element_type_f32_0 = hal.element_type<f32> : i32 | |
%dense_row_major_1 = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32_0) encoding(%dense_row_major_1) | |
%3 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%4 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%3} | |
%5 = stream.async.transfer %4 : !stream.resource<external>{%3} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%3} | |
%6 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%6} | |
%8 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%9 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%5[%c0 to %3 for %3]) : (!stream.resource<*>{%3}) -> !stream.resource<*>{%8} | |
%10 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7[%c0 to %6 for %6], %9[%c0 to %8 for %8]) : (!stream.resource<*>{%6}, !stream.resource<*>{%8}) -> !stream.resource<*>{%10} | |
%12 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%13 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%11[%c0 to %10 for %10]) : (!stream.resource<*>{%10}) -> !stream.resource<*>{%12} | |
%14 = stream.async.transfer %13 : !stream.resource<*>{%12} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%12} | |
%15 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %14 : tensor<64x64xf32> in !stream.resource<external>{%12} -> !hal.buffer_view | |
util.return %15 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CombineInitializers (iree-util-combine-initializers) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EncodeDeviceTensorsPass (iree-stream-encode-device-tensors) ('stream.executable' operation: @matmul_dispatch_0) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EncodeDeviceTensorsPass (iree-stream-encode-device-tensors) ('stream.executable' operation: @matmul_dispatch_1) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EncodeDeviceTensorsPass (iree-stream-encode-device-tensors) ('stream.executable' operation: @matmul_dispatch_2) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EncodeDeviceTensorsPass (iree-stream-encode-device-tensors) ('stream.executable' operation: @matmul_dispatch_3) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<64x64xf32> : index | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%2 = stream.async.transfer %1 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%3 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%0} | |
%4 = stream.async.transfer %3 : !stream.resource<external>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%0} | |
%5 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x64x8x1xf32> : index | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%2[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%5} | |
%7 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<16x64x4x1xf32> : index | |
%8 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%4[%c0 to %0 for %0]) : (!stream.resource<*>{%0}) -> !stream.resource<*>{%7} | |
%9 = stream.tensor.sizeof on(#hal.device.affinity<@__device_0>) tensor<8x16x8x4xf32> : index | |
%10 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%6[%c0 to %5 for %5], %8[%c0 to %7 for %7]) : (!stream.resource<*>{%5}, !stream.resource<*>{%7}) -> !stream.resource<*>{%9} | |
%11 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%10[%c0 to %9 for %9]) : (!stream.resource<*>{%9}) -> !stream.resource<*>{%0} | |
%12 = stream.async.transfer %11 : !stream.resource<*>{%0} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%0} | |
%13 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %12 : tensor<64x64xf32> in !stream.resource<external>{%0} -> !hal.buffer_view | |
util.return %13 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EncodeHostTensorsPass (iree-stream-encode-host-tensors) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyLoweringToAsyncResourcesPass (iree-stream-verify-lowering-to-async-resources) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After MaterializeCopyOnWritePass (iree-stream-materialize-copy-on-write) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ElideAsyncCopiesPass (iree-stream-elide-async-copies) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After EmplaceAllocationsPass (iree-stream-emplace-allocations) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%1 = stream.async.transfer %0 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%2 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%3 = stream.async.transfer %2 : !stream.resource<external>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<*>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%3[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%6 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}, !stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%7 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<*>{%c16384}) -> !stream.resource<*>{%c16384} | |
%8 = stream.async.transfer %7 : !stream.resource<*>{%c16384} from(#hal.device.affinity<@__device_0>) -> to(#hal.device.affinity<@__device_0>) !stream.resource<external>{%c16384} | |
%9 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %8 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %9 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After RefineUsagePass (iree-stream-refine-usage) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyAsyncAccessRangesPass (iree-stream-verify-async-access-ranges) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%0[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%3 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%1[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%4 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%2[%c0 to %c16384 for %c16384], %3[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch on(#hal.device.affinity<@__device_0>) @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%4[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ScheduleExecutionPass (iree-stream-schedule-execution) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg2[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%5 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg3[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4[%c0 to %c16384 for %c16384], %5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%7 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%6[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %7 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ScheduleConcurrencyPass (iree-stream-schedule-concurrency) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After PropagateTimepointsPass (iree-stream-propagate-timepoints) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.timepoint.immediate => !stream.timepoint | |
%3 = stream.timepoint.immediate => !stream.timepoint | |
%4 = stream.timepoint.join max(%2, %3) => !stream.timepoint | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) await(%4) => with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%7:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%10 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%11 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %10, %11 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%8 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7#0[%c0 to %c16384 for %c16384], %7#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%9 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%8[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %9 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%5 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After MaterializeBuiltinsPass (iree-stream-materialize-builtins) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%2 = stream.timepoint.immediate => !stream.timepoint | |
%3 = stream.timepoint.immediate => !stream.timepoint | |
%4 = stream.timepoint.join max(%2, %3) => !stream.timepoint | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) await(%4) => with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%7:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%10 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%11 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %10, %11 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%8 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%7#0[%c0 to %c16384 for %c16384], %7#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%9 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%8[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %9 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%5 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%6 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %5 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %6 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After Canonicalizer (canonicalize) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After CSE (cse) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After SimplifyGlobalAccesses (iree-util-simplify-global-accesses) ('util.func' operation: @matmul) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ApplyPatterns (iree-util-apply-patterns) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FoldGlobals (iree-util-fold-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After FuseGlobals (iree-util-fuse-globals) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After IPO (iree-util-ipo) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After VerifyLoweringToAsyncPass (iree-stream-verify-lowering-to-async) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<16x64x4x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [1, 0] inner_dims_pos = [1, 0] inner_tiles = [4, 1] into %3 : tensor<64x64xf32> -> tensor<16x64x4x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : tensor<16x64x4x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<16x64x4x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_2 { | |
stream.executable.export public @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%arg0: !stream.binding, %arg1: !stream.binding, %arg2: !stream.binding) { | |
%cst = arith.constant 0.000000e+00 : f32 | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> | |
%2 = stream.binding.subspan %arg2[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
%3 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x64x8x1xf32>> -> tensor<8x64x8x1xf32> | |
%4 = flow.dispatch.tensor.load %1, offsets = [0, 0, 0, 0], sizes = [16, 64, 4, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<16x64x4x1xf32>> -> tensor<16x64x4x1xf32> | |
%5 = tensor.empty() : tensor<8x16x8x4xf32> | |
%6 = linalg.fill ins(%cst : f32) outs(%5 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
%7 = linalg.mmt4d ins(%3, %4 : tensor<8x64x8x1xf32>, tensor<16x64x4x1xf32>) outs(%6 : tensor<8x16x8x4xf32>) -> tensor<8x16x8x4xf32> | |
flow.dispatch.tensor.store %7, %2, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : tensor<8x16x8x4xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x16x8x4xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_3 { | |
stream.executable.export public @matmul_dispatch_3_unpack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_3_unpack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [8, 16, 8, 4], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<8x16x8x4xf32>> -> tensor<8x16x8x4xf32> | |
%3 = tensor.empty() : tensor<64x64xf32> | |
%unpack = tensor.unpack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 4] into %3 : tensor<8x16x8x4xf32> -> tensor<64x64xf32> | |
flow.dispatch.tensor.store %unpack, %1, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : tensor<64x64xf32> -> !flow.dispatch.tensor<writeonly:tensor<64x64xf32>> | |
return | |
} | |
} | |
} | |
util.func public @matmul(%arg0: !hal.buffer_view, %arg1: !hal.buffer_view) -> !hal.buffer_view attributes {iree.abi.stub, iree.reflection = {iree.abi.declaration = "sync func @matmul(%input0: tensor<64x64xf32>, %input1: tensor<64x64xf32>) -> (%output0: tensor<64x64xf32>)"}} { | |
%c16384 = arith.constant 16384 : index | |
%c0 = arith.constant 0 : index | |
%c64 = arith.constant 64 : index | |
%element_type_f32 = hal.element_type<f32> : i32 | |
%dense_row_major = hal.encoding_type<dense_row_major> : i32 | |
hal.buffer_view.assert<%arg0 : !hal.buffer_view> message("input0") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%0 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg0 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
hal.buffer_view.assert<%arg1 : !hal.buffer_view> message("input1") shape([%c64, %c64]) type(%element_type_f32) encoding(%dense_row_major) | |
%1 = stream.tensor.import on(#hal.device.affinity<@__device_0>) %arg1 : !hal.buffer_view -> tensor<64x64xf32> in !stream.resource<external>{%c16384} | |
%results, %result_timepoint = stream.async.execute on(#hal.device.affinity<@__device_0>) with(%0 as %arg2: !stream.resource<external>{%c16384}, %1 as %arg3: !stream.resource<external>{%c16384}) -> !stream.resource<external>{%c16384} { | |
%4:2 = stream.async.concurrent with(%arg2 as %arg4: !stream.resource<external>{%c16384}, %arg3 as %arg5: !stream.resource<external>{%c16384}) -> (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) { | |
%7 = stream.async.dispatch @matmul_dispatch_0::@matmul_dispatch_0_pack_f32(%arg4[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%8 = stream.async.dispatch @matmul_dispatch_1::@matmul_dispatch_1_pack_f32(%arg5[%c0 to %c16384 for %c16384]) : (!stream.resource<external>{%c16384}) -> !stream.resource<transient>{%c16384} | |
stream.yield %7, %8 : !stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384} | |
} | |
%5 = stream.async.dispatch @matmul_dispatch_2::@matmul_dispatch_2_mmt4d_8x16x64x8x4x1_f32(%4#0[%c0 to %c16384 for %c16384], %4#1[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}, !stream.resource<transient>{%c16384}) -> !stream.resource<transient>{%c16384} | |
%6 = stream.async.dispatch @matmul_dispatch_3::@matmul_dispatch_3_unpack_f32(%5[%c0 to %c16384 for %c16384]) : (!stream.resource<transient>{%c16384}) -> !stream.resource<external>{%c16384} | |
stream.yield %6 : !stream.resource<external>{%c16384} | |
} => !stream.timepoint | |
%2 = stream.timepoint.await %result_timepoint => %results : !stream.resource<external>{%c16384} | |
%3 = stream.tensor.export on(#hal.device.affinity<@__device_0>) %2 : tensor<64x64xf32> in !stream.resource<external>{%c16384} -> !hal.buffer_view | |
util.return %3 : !hal.buffer_view | |
} | |
} | |
// -----// IR Dump After ScheduleAllocationPass (iree-stream-schedule-allocation) ('builtin.module' operation) //----- // | |
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", native_vector_size = 16 : i64, target_triple = "x86_64-unknown-unknown-eabi-elf"}> | |
#device_target_local = #hal.device.target<"local", [#executable_target_embedded_elf_x86_64_]> : !hal.device | |
module attributes {stream.affinity.default = #hal.device.affinity<@__device_0>} { | |
util.global private @__device_0 = #device_target_local | |
stream.executable private @matmul_dispatch_0 { | |
stream.executable.export public @matmul_dispatch_0_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_0_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stream.binding -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
%2 = flow.dispatch.tensor.load %0, offsets = [0, 0], sizes = [64, 64], strides = [1, 1] : !flow.dispatch.tensor<readonly:tensor<64x64xf32>> -> tensor<64x64xf32> | |
%3 = tensor.empty() : tensor<8x64x8x1xf32> | |
%pack = tensor.pack %2 outer_dims_perm = [0, 1] inner_dims_pos = [0, 1] inner_tiles = [8, 1] into %3 : tensor<64x64xf32> -> tensor<8x64x8x1xf32> | |
flow.dispatch.tensor.store %pack, %1, offsets = [0, 0, 0, 0], sizes = [8, 64, 8, 1], strides = [1, 1, 1, 1] : tensor<8x64x8x1xf32> -> !flow.dispatch.tensor<writeonly:tensor<8x64x8x1xf32>> | |
return | |
} | |
} | |
} | |
stream.executable private @matmul_dispatch_1 { | |
stream.executable.export public @matmul_dispatch_1_pack_f32 workgroups() -> (index, index, index) { | |
%x, %y, %z = flow.dispatch.workgroup_count_from_slice | |
stream.return %x, %y, %z : index, index, index | |
} | |
builtin.module { | |
func.func @matmul_dispatch_1_pack_f32(%arg0: !stream.binding, %arg1: !stream.binding) { | |
%c0 = arith.constant 0 : index | |
%0 = stream.binding.subspan %arg0[%c0] : !stream.binding -> !flow.dispatch.tensor<readonly:tensor<64x64xf32>> | |
%1 = stream.binding.subspan %arg1[%c0] : !stre |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment