Skip to content

Instantly share code, notes, and snippets.

@AmosLewis

AmosLewis/distilgpt_torch_raw_transformers4.21.2_elided_torchbackend_tosabackend_elide_Canonicalizer.mlir

Created January 17, 2023 19:27
Show Gist options
  • Save AmosLewis/f69629625defdcd9105c3216226e5e9f to your computer and use it in GitHub Desktop.
Save AmosLewis/f69629625defdcd9105c3216226e5e9f to your computer and use it in GitHub Desktop.
Download ZIP
Raw
distilgpt_torch_raw_transformers4.21.2_elided_torchbackend_tosabackend_elide_Canonicalizer.mlir
module attributes {torch.debug_module_name = "_lambda"} {
func.func @forward(%arg0: tensor<1x128xi64>) -> tensor<1x2xf32> {
%0 = "tosa.const"() {value = dense_resource<__elided__> : tensor<2x768xf32>} : () -> tensor<2x768xf32>
%1 = "tosa.const"() {value = dense_resource<__elided__> : tensor<3072x768xf32>} : () -> tensor<3072x768xf32>
%2 = "tosa.const"() {value = dense_resource<__elided__> : tensor<768x3072xf32>} : () -> tensor<768x3072xf32>
%3 = "tosa.const"() {value = dense_resource<__elided__> : tensor<3072xf32>} : () -> tensor<3072xf32>
%4 = "tosa.const"() {value = dense_resource<__elided__> : tensor<768x768xf32>} : () -> tensor<768x768xf32>
%5 = "tosa.const"() {value = dense<-3.40282347E+38> : tensor<f32>} : () -> tensor<f32>
%6 = "tosa.const"() {value = dense_resource<__elided__> : tensor<1x1x1024x1024xui8>} : () -> tensor<1x1x1024x1024xi8>
%7 = "tosa.const"() {value = dense<8.000000e+00> : tensor<f32>} : () -> tensor<f32>
%8 = "tosa.const"() {value = dense_resource<__elided__> : tensor<768x2304xf32>} : () -> tensor<768x2304xf32>
%9 = "tosa.const"() {value = dense_resource<__elided__> : tensor<2304xf32>} : () -> tensor<2304xf32>
%10 = "tosa.const"() {value = dense_resource<__elided__> : tensor<768xf32>} : () -> tensor<768xf32>
%11 = "tosa.const"() {value = dense_resource<__elided__> : tensor<1024x768xf32>} : () -> tensor<1024x768xf32>
%12 = "tosa.const"() {value = dense_resource<__elided__> : tensor<50257x768xf32>} : () -> tensor<50257x768xf32>
%13 = "tosa.const"() {value = dense_resource<__elided__> : tensor<128xi64>} : () -> tensor<128xi64>
%14 = "tosa.const"() {value = dense<7.680000e+02> : tensor<f32>} : () -> tensor<f32>
%15 = "tosa.const"() {value = dense_resource<__elided__> : tensor<1x98304xi32>} : () -> tensor<1x98304xi32>
%16 = "tosa.const"() {value = dense<[0, 2, 1, 3]> : tensor<4xi64>} : () -> tensor<4xi64>
%17 = "tosa.const"() {value = dense<[0, 1, 3, 2]> : tensor<4xi32>} : () -> tensor<4xi32>
%18 = "tosa.const"() {value = dense_resource<__elided__> : tensor<1x1048576xi32>} : () -> tensor<1x1048576xi32>
%19 = "tosa.const"() {value = dense_resource<__elided__> : tensor<1x131072xi32>} : () -> tensor<1x131072xi32>
%20 = "tosa.const"() {value = dense_resource<__elided__> : tensor<1x16384xi32>} : () -> tensor<1x16384xi32>
%21 = "tosa.const"() {value = dense<0> : tensor<1x1x128x128xi8>} : () -> tensor<1x1x128x128xi8>
%22 = "tosa.const"() {value = dense<[1, 0]> : tensor<2xi32>} : () -> tensor<2xi32>
%23 = "tosa.const"() {value = dense<0> : tensor<1xi64>} : () -> tensor<1xi64>
%24 = "tosa.const"() {value = dense<0.797884583> : tensor<1x1x1xf32>} : () -> tensor<1x1x1xf32>
%25 = "tosa.const"() {value = dense<4.471500e-02> : tensor<1x1x1xf32>} : () -> tensor<1x1x1xf32>
%26 = "tosa.const"() {value = dense<3.000000e+00> : tensor<1x1x1xf32>} : () -> tensor<1x1x1xf32>
%27 = "tosa.const"() {value = dense<5.000000e-01> : tensor<1x1x1xf32>} : () -> tensor<1x1x1xf32>
%28 = "tosa.const"() {value = dense<9.99999974E-6> : tensor<1x1x1xf32>} : () -> tensor<1x1x1xf32>
%29 = "tosa.const"() {value = dense<1.000000e+00> : tensor<1x1x1xf32>} : () -> tensor<1x1x1xf32>
%30 = "tosa.reshape"(%13) {new_shape = array<i64: 1, 128>} : (tensor<128xi64>) -> tensor<1x128xi64>
%31 = "tosa.reshape"(%12) {new_shape = array<i64: 1, 50257, 768>} : (tensor<50257x768xf32>) -> tensor<1x50257x768xf32>
%32 = "tosa.cast"(%arg0) : (tensor<1x128xi64>) -> tensor<1x128xi32>
%33 = "tosa.gather"(%31, %32) : (tensor<1x50257x768xf32>, tensor<1x128xi32>) -> tensor<1x128x768xf32>
%34 = "tosa.reshape"(%11) {new_shape = array<i64: 1, 1024, 768>} : (tensor<1024x768xf32>) -> tensor<1x1024x768xf32>
%35 = "tosa.cast"(%30) : (tensor<1x128xi64>) -> tensor<1x128xi32>
%36 = "tosa.gather"(%34, %35) : (tensor<1x1024x768xf32>, tensor<1x128xi32>) -> tensor<1x128x768xf32>
%37 = "tosa.add"(%33, %36) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%38 = "tosa.reduce_sum"(%37) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%39 = "tosa.reciprocal"(%14) : (tensor<f32>) -> tensor<f32>
%40 = "tosa.reshape"(%39) {new_shape = array<i64: 1, 1, 1>} : (tensor<f32>) -> tensor<1x1x1xf32>
%41 = "tosa.mul"(%38, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%42 = "tosa.sub"(%37, %41) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%43 = "tosa.mul"(%42, %42) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%44 = "tosa.reduce_sum"(%43) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%45 = "tosa.mul"(%44, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%46 = "tosa.add"(%45, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%47 = "tosa.rsqrt"(%46) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%48 = "tosa.mul"(%42, %47) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%49 = "tosa.reshape"(%10) {new_shape = array<i64: 1, 1, 768>} : (tensor<768xf32>) -> tensor<1x1x768xf32>
%50 = "tosa.mul"(%48, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%51 = "tosa.add"(%50, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%52 = "tosa.reshape"(%8) {new_shape = array<i64: 1, 768, 2304>} : (tensor<768x2304xf32>) -> tensor<1x768x2304xf32>
%53 = "tosa.matmul"(%51, %52) : (tensor<1x128x768xf32>, tensor<1x768x2304xf32>) -> tensor<1x128x2304xf32>
%54 = "tosa.reshape"(%53) {new_shape = array<i64: 128, 2304>} : (tensor<1x128x2304xf32>) -> tensor<128x2304xf32>
%55 = "tosa.reshape"(%9) {new_shape = array<i64: 1, 2304>} : (tensor<2304xf32>) -> tensor<1x2304xf32>
%56 = "tosa.add"(%55, %54) : (tensor<1x2304xf32>, tensor<128x2304xf32>) -> tensor<128x2304xf32>
%57 = "tosa.reshape"(%56) {new_shape = array<i64: 1, 294912, 1>} : (tensor<128x2304xf32>) -> tensor<1x294912x1xf32>
%58 = "tosa.gather"(%57, %15) : (tensor<1x294912x1xf32>, tensor<1x98304xi32>) -> tensor<1x98304x1xf32>
%59 = "tosa.reshape"(%58) {new_shape = array<i64: 1, 128, 12, 64>} : (tensor<1x98304x1xf32>) -> tensor<1x128x12x64xf32>
%60 = "tosa.transpose"(%59, %16) : (tensor<1x128x12x64xf32>, tensor<4xi64>) -> tensor<1x12x128x64xf32>
%61 = "tosa.transpose"(%60, %17) : (tensor<1x12x128x64xf32>, tensor<4xi32>) -> tensor<1x12x64x128xf32>
%62 = "tosa.reshape"(%60) {new_shape = array<i64: 12, 128, 64>} : (tensor<1x12x128x64xf32>) -> tensor<12x128x64xf32>
%63 = "tosa.reshape"(%61) {new_shape = array<i64: 12, 64, 128>} : (tensor<1x12x64x128xf32>) -> tensor<12x64x128xf32>
%64 = "tosa.matmul"(%62, %63) : (tensor<12x128x64xf32>, tensor<12x64x128xf32>) -> tensor<12x128x128xf32>
%65 = "tosa.reshape"(%64) {new_shape = array<i64: 1, 12, 128, 128>} : (tensor<12x128x128xf32>) -> tensor<1x12x128x128xf32>
%66 = "tosa.reciprocal"(%7) : (tensor<f32>) -> tensor<f32>
%67 = "tosa.reshape"(%66) {new_shape = array<i64: 1, 1, 1, 1>} : (tensor<f32>) -> tensor<1x1x1x1xf32>
%68 = "tosa.mul"(%65, %67) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x1x1x1xf32>) -> tensor<1x12x128x128xf32>
%69 = "tosa.reshape"(%6) {new_shape = array<i64: 1, 1048576, 1>} : (tensor<1x1x1024x1024xi8>) -> tensor<1x1048576x1xi8>
%70 = "tosa.gather"(%69, %18) : (tensor<1x1048576x1xi8>, tensor<1x1048576xi32>) -> tensor<1x1048576x1xi8>
%71 = "tosa.gather"(%70, %18) : (tensor<1x1048576x1xi8>, tensor<1x1048576xi32>) -> tensor<1x1048576x1xi8>
%72 = "tosa.gather"(%71, %19) : (tensor<1x1048576x1xi8>, tensor<1x131072xi32>) -> tensor<1x131072x1xi8>
%73 = "tosa.gather"(%72, %20) : (tensor<1x131072x1xi8>, tensor<1x16384xi32>) -> tensor<1x16384x1xi8>
%74 = "tosa.reshape"(%73) {new_shape = array<i64: 1, 1, 128, 128>} : (tensor<1x16384x1xi8>) -> tensor<1x1x128x128xi8>
%75 = "tosa.equal"(%74, %21) : (tensor<1x1x128x128xi8>, tensor<1x1x128x128xi8>) -> tensor<1x1x128x128xi1>
%76 = "tosa.select"(%75, %5, %68) : (tensor<1x1x128x128xi1>, tensor<f32>, tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%77 = "tosa.reduce_max"(%76) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%78 = "tosa.sub"(%76, %77) : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%79 = "tosa.exp"(%78) : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%80 = "tosa.reduce_sum"(%79) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%81 = "tosa.reciprocal"(%80) : (tensor<1x12x128x1xf32>) -> tensor<1x12x128x1xf32>
%82 = "tosa.mul"(%79, %81) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%83 = "tosa.reshape"(%82) {new_shape = array<i64: 12, 128, 128>} : (tensor<1x12x128x128xf32>) -> tensor<12x128x128xf32>
%84 = "tosa.matmul"(%83, %62) : (tensor<12x128x128xf32>, tensor<12x128x64xf32>) -> tensor<12x128x64xf32>
%85 = "tosa.reshape"(%84) {new_shape = array<i64: 1, 12, 128, 64>} : (tensor<12x128x64xf32>) -> tensor<1x12x128x64xf32>
%86 = "tosa.transpose"(%85, %16) : (tensor<1x12x128x64xf32>, tensor<4xi64>) -> tensor<1x128x12x64xf32>
%87 = "tosa.reshape"(%86) {new_shape = array<i64: 1, 128, 768>} : (tensor<1x128x12x64xf32>) -> tensor<1x128x768xf32>
%88 = "tosa.reshape"(%4) {new_shape = array<i64: 1, 768, 768>} : (tensor<768x768xf32>) -> tensor<1x768x768xf32>
%89 = "tosa.matmul"(%87, %88) : (tensor<1x128x768xf32>, tensor<1x768x768xf32>) -> tensor<1x128x768xf32>
%90 = "tosa.reshape"(%89) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%91 = "tosa.reshape"(%10) {new_shape = array<i64: 1, 768>} : (tensor<768xf32>) -> tensor<1x768xf32>
%92 = "tosa.add"(%91, %90) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%93 = "tosa.reshape"(%92) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%94 = "tosa.add"(%93, %37) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%95 = "tosa.reduce_sum"(%94) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%96 = "tosa.mul"(%95, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%97 = "tosa.sub"(%94, %96) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%98 = "tosa.mul"(%97, %97) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%99 = "tosa.reduce_sum"(%98) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%100 = "tosa.mul"(%99, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%101 = "tosa.add"(%100, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%102 = "tosa.rsqrt"(%101) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%103 = "tosa.mul"(%97, %102) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%104 = "tosa.mul"(%103, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%105 = "tosa.add"(%104, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%106 = "tosa.reshape"(%2) {new_shape = array<i64: 1, 768, 3072>} : (tensor<768x3072xf32>) -> tensor<1x768x3072xf32>
%107 = "tosa.matmul"(%105, %106) : (tensor<1x128x768xf32>, tensor<1x768x3072xf32>) -> tensor<1x128x3072xf32>
%108 = "tosa.reshape"(%107) {new_shape = array<i64: 128, 3072>} : (tensor<1x128x3072xf32>) -> tensor<128x3072xf32>
%109 = "tosa.reshape"(%3) {new_shape = array<i64: 1, 3072>} : (tensor<3072xf32>) -> tensor<1x3072xf32>
%110 = "tosa.add"(%109, %108) : (tensor<1x3072xf32>, tensor<128x3072xf32>) -> tensor<128x3072xf32>
%111 = "tosa.reshape"(%110) {new_shape = array<i64: 1, 128, 3072>} : (tensor<128x3072xf32>) -> tensor<1x128x3072xf32>
%112 = "tosa.mul"(%111, %27) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%113 = "tosa.pow"(%111, %26) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%114 = "tosa.mul"(%113, %25) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%115 = "tosa.add"(%111, %114) : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%116 = "tosa.mul"(%115, %24) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%117 = "tosa.tanh"(%116) : (tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%118 = "tosa.add"(%117, %29) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%119 = "tosa.mul"(%112, %118) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%120 = "tosa.reshape"(%1) {new_shape = array<i64: 1, 3072, 768>} : (tensor<3072x768xf32>) -> tensor<1x3072x768xf32>
%121 = "tosa.matmul"(%119, %120) : (tensor<1x128x3072xf32>, tensor<1x3072x768xf32>) -> tensor<1x128x768xf32>
%122 = "tosa.reshape"(%121) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%123 = "tosa.add"(%91, %122) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%124 = "tosa.reshape"(%123) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%125 = "tosa.add"(%94, %124) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%126 = "tosa.reduce_sum"(%125) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%127 = "tosa.mul"(%126, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%128 = "tosa.sub"(%125, %127) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%129 = "tosa.mul"(%128, %128) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%130 = "tosa.reduce_sum"(%129) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%131 = "tosa.mul"(%130, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%132 = "tosa.add"(%131, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%133 = "tosa.rsqrt"(%132) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%134 = "tosa.mul"(%128, %133) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%135 = "tosa.mul"(%134, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%136 = "tosa.add"(%135, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%137 = "tosa.matmul"(%136, %52) : (tensor<1x128x768xf32>, tensor<1x768x2304xf32>) -> tensor<1x128x2304xf32>
%138 = "tosa.reshape"(%137) {new_shape = array<i64: 128, 2304>} : (tensor<1x128x2304xf32>) -> tensor<128x2304xf32>
%139 = "tosa.add"(%55, %138) : (tensor<1x2304xf32>, tensor<128x2304xf32>) -> tensor<128x2304xf32>
%140 = "tosa.reshape"(%139) {new_shape = array<i64: 1, 294912, 1>} : (tensor<128x2304xf32>) -> tensor<1x294912x1xf32>
%141 = "tosa.gather"(%140, %15) : (tensor<1x294912x1xf32>, tensor<1x98304xi32>) -> tensor<1x98304x1xf32>
%142 = "tosa.reshape"(%141) {new_shape = array<i64: 1, 128, 12, 64>} : (tensor<1x98304x1xf32>) -> tensor<1x128x12x64xf32>
%143 = "tosa.transpose"(%142, %16) : (tensor<1x128x12x64xf32>, tensor<4xi64>) -> tensor<1x12x128x64xf32>
%144 = "tosa.transpose"(%143, %17) : (tensor<1x12x128x64xf32>, tensor<4xi32>) -> tensor<1x12x64x128xf32>
%145 = "tosa.reshape"(%143) {new_shape = array<i64: 12, 128, 64>} : (tensor<1x12x128x64xf32>) -> tensor<12x128x64xf32>
%146 = "tosa.reshape"(%144) {new_shape = array<i64: 12, 64, 128>} : (tensor<1x12x64x128xf32>) -> tensor<12x64x128xf32>
%147 = "tosa.matmul"(%145, %146) : (tensor<12x128x64xf32>, tensor<12x64x128xf32>) -> tensor<12x128x128xf32>
%148 = "tosa.reshape"(%147) {new_shape = array<i64: 1, 12, 128, 128>} : (tensor<12x128x128xf32>) -> tensor<1x12x128x128xf32>
%149 = "tosa.mul"(%148, %67) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x1x1x1xf32>) -> tensor<1x12x128x128xf32>
%150 = "tosa.select"(%75, %5, %149) : (tensor<1x1x128x128xi1>, tensor<f32>, tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%151 = "tosa.reduce_max"(%150) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%152 = "tosa.sub"(%150, %151) : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%153 = "tosa.exp"(%152) : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%154 = "tosa.reduce_sum"(%153) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%155 = "tosa.reciprocal"(%154) : (tensor<1x12x128x1xf32>) -> tensor<1x12x128x1xf32>
%156 = "tosa.mul"(%153, %155) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%157 = "tosa.reshape"(%156) {new_shape = array<i64: 12, 128, 128>} : (tensor<1x12x128x128xf32>) -> tensor<12x128x128xf32>
%158 = "tosa.matmul"(%157, %145) : (tensor<12x128x128xf32>, tensor<12x128x64xf32>) -> tensor<12x128x64xf32>
%159 = "tosa.reshape"(%158) {new_shape = array<i64: 1, 12, 128, 64>} : (tensor<12x128x64xf32>) -> tensor<1x12x128x64xf32>
%160 = "tosa.transpose"(%159, %16) : (tensor<1x12x128x64xf32>, tensor<4xi64>) -> tensor<1x128x12x64xf32>
%161 = "tosa.reshape"(%160) {new_shape = array<i64: 1, 128, 768>} : (tensor<1x128x12x64xf32>) -> tensor<1x128x768xf32>
%162 = "tosa.matmul"(%161, %88) : (tensor<1x128x768xf32>, tensor<1x768x768xf32>) -> tensor<1x128x768xf32>
%163 = "tosa.reshape"(%162) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%164 = "tosa.add"(%91, %163) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%165 = "tosa.reshape"(%164) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%166 = "tosa.add"(%165, %125) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%167 = "tosa.reduce_sum"(%166) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%168 = "tosa.mul"(%167, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%169 = "tosa.sub"(%166, %168) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%170 = "tosa.mul"(%169, %169) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%171 = "tosa.reduce_sum"(%170) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%172 = "tosa.mul"(%171, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%173 = "tosa.add"(%172, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%174 = "tosa.rsqrt"(%173) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%175 = "tosa.mul"(%169, %174) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%176 = "tosa.mul"(%175, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%177 = "tosa.add"(%176, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%178 = "tosa.matmul"(%177, %106) : (tensor<1x128x768xf32>, tensor<1x768x3072xf32>) -> tensor<1x128x3072xf32>
%179 = "tosa.reshape"(%178) {new_shape = array<i64: 128, 3072>} : (tensor<1x128x3072xf32>) -> tensor<128x3072xf32>
%180 = "tosa.add"(%109, %179) : (tensor<1x3072xf32>, tensor<128x3072xf32>) -> tensor<128x3072xf32>
%181 = "tosa.reshape"(%180) {new_shape = array<i64: 1, 128, 3072>} : (tensor<128x3072xf32>) -> tensor<1x128x3072xf32>
%182 = "tosa.mul"(%181, %27) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%183 = "tosa.pow"(%181, %26) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%184 = "tosa.mul"(%183, %25) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%185 = "tosa.add"(%181, %184) : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%186 = "tosa.mul"(%185, %24) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%187 = "tosa.tanh"(%186) : (tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%188 = "tosa.add"(%187, %29) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%189 = "tosa.mul"(%182, %188) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%190 = "tosa.matmul"(%189, %120) : (tensor<1x128x3072xf32>, tensor<1x3072x768xf32>) -> tensor<1x128x768xf32>
%191 = "tosa.reshape"(%190) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%192 = "tosa.add"(%91, %191) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%193 = "tosa.reshape"(%192) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%194 = "tosa.add"(%166, %193) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%195 = "tosa.reduce_sum"(%194) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%196 = "tosa.mul"(%195, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%197 = "tosa.sub"(%194, %196) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%198 = "tosa.mul"(%197, %197) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%199 = "tosa.reduce_sum"(%198) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%200 = "tosa.mul"(%199, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%201 = "tosa.add"(%200, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%202 = "tosa.rsqrt"(%201) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%203 = "tosa.mul"(%197, %202) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%204 = "tosa.mul"(%203, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%205 = "tosa.add"(%204, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%206 = "tosa.matmul"(%205, %52) : (tensor<1x128x768xf32>, tensor<1x768x2304xf32>) -> tensor<1x128x2304xf32>
%207 = "tosa.reshape"(%206) {new_shape = array<i64: 128, 2304>} : (tensor<1x128x2304xf32>) -> tensor<128x2304xf32>
%208 = "tosa.add"(%55, %207) : (tensor<1x2304xf32>, tensor<128x2304xf32>) -> tensor<128x2304xf32>
%209 = "tosa.reshape"(%208) {new_shape = array<i64: 1, 294912, 1>} : (tensor<128x2304xf32>) -> tensor<1x294912x1xf32>
%210 = "tosa.gather"(%209, %15) : (tensor<1x294912x1xf32>, tensor<1x98304xi32>) -> tensor<1x98304x1xf32>
%211 = "tosa.reshape"(%210) {new_shape = array<i64: 1, 128, 12, 64>} : (tensor<1x98304x1xf32>) -> tensor<1x128x12x64xf32>
%212 = "tosa.transpose"(%211, %16) : (tensor<1x128x12x64xf32>, tensor<4xi64>) -> tensor<1x12x128x64xf32>
%213 = "tosa.transpose"(%212, %17) : (tensor<1x12x128x64xf32>, tensor<4xi32>) -> tensor<1x12x64x128xf32>
%214 = "tosa.reshape"(%212) {new_shape = array<i64: 12, 128, 64>} : (tensor<1x12x128x64xf32>) -> tensor<12x128x64xf32>
%215 = "tosa.reshape"(%213) {new_shape = array<i64: 12, 64, 128>} : (tensor<1x12x64x128xf32>) -> tensor<12x64x128xf32>
%216 = "tosa.matmul"(%214, %215) : (tensor<12x128x64xf32>, tensor<12x64x128xf32>) -> tensor<12x128x128xf32>
%217 = "tosa.reshape"(%216) {new_shape = array<i64: 1, 12, 128, 128>} : (tensor<12x128x128xf32>) -> tensor<1x12x128x128xf32>
%218 = "tosa.mul"(%217, %67) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x1x1x1xf32>) -> tensor<1x12x128x128xf32>
%219 = "tosa.select"(%75, %5, %218) : (tensor<1x1x128x128xi1>, tensor<f32>, tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%220 = "tosa.reduce_max"(%219) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%221 = "tosa.sub"(%219, %220) : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%222 = "tosa.exp"(%221) : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%223 = "tosa.reduce_sum"(%222) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%224 = "tosa.reciprocal"(%223) : (tensor<1x12x128x1xf32>) -> tensor<1x12x128x1xf32>
%225 = "tosa.mul"(%222, %224) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%226 = "tosa.reshape"(%225) {new_shape = array<i64: 12, 128, 128>} : (tensor<1x12x128x128xf32>) -> tensor<12x128x128xf32>
%227 = "tosa.matmul"(%226, %214) : (tensor<12x128x128xf32>, tensor<12x128x64xf32>) -> tensor<12x128x64xf32>
%228 = "tosa.reshape"(%227) {new_shape = array<i64: 1, 12, 128, 64>} : (tensor<12x128x64xf32>) -> tensor<1x12x128x64xf32>
%229 = "tosa.transpose"(%228, %16) : (tensor<1x12x128x64xf32>, tensor<4xi64>) -> tensor<1x128x12x64xf32>
%230 = "tosa.reshape"(%229) {new_shape = array<i64: 1, 128, 768>} : (tensor<1x128x12x64xf32>) -> tensor<1x128x768xf32>
%231 = "tosa.matmul"(%230, %88) : (tensor<1x128x768xf32>, tensor<1x768x768xf32>) -> tensor<1x128x768xf32>
%232 = "tosa.reshape"(%231) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%233 = "tosa.add"(%91, %232) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%234 = "tosa.reshape"(%233) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%235 = "tosa.add"(%234, %194) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%236 = "tosa.reduce_sum"(%235) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%237 = "tosa.mul"(%236, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%238 = "tosa.sub"(%235, %237) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%239 = "tosa.mul"(%238, %238) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%240 = "tosa.reduce_sum"(%239) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%241 = "tosa.mul"(%240, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%242 = "tosa.add"(%241, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%243 = "tosa.rsqrt"(%242) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%244 = "tosa.mul"(%238, %243) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%245 = "tosa.mul"(%244, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%246 = "tosa.add"(%245, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%247 = "tosa.matmul"(%246, %106) : (tensor<1x128x768xf32>, tensor<1x768x3072xf32>) -> tensor<1x128x3072xf32>
%248 = "tosa.reshape"(%247) {new_shape = array<i64: 128, 3072>} : (tensor<1x128x3072xf32>) -> tensor<128x3072xf32>
%249 = "tosa.add"(%109, %248) : (tensor<1x3072xf32>, tensor<128x3072xf32>) -> tensor<128x3072xf32>
%250 = "tosa.reshape"(%249) {new_shape = array<i64: 1, 128, 3072>} : (tensor<128x3072xf32>) -> tensor<1x128x3072xf32>
%251 = "tosa.mul"(%250, %27) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%252 = "tosa.pow"(%250, %26) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%253 = "tosa.mul"(%252, %25) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%254 = "tosa.add"(%250, %253) : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%255 = "tosa.mul"(%254, %24) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%256 = "tosa.tanh"(%255) : (tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%257 = "tosa.add"(%256, %29) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%258 = "tosa.mul"(%251, %257) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%259 = "tosa.matmul"(%258, %120) : (tensor<1x128x3072xf32>, tensor<1x3072x768xf32>) -> tensor<1x128x768xf32>
%260 = "tosa.reshape"(%259) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%261 = "tosa.add"(%91, %260) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%262 = "tosa.reshape"(%261) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%263 = "tosa.add"(%235, %262) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%264 = "tosa.reduce_sum"(%263) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%265 = "tosa.mul"(%264, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%266 = "tosa.sub"(%263, %265) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%267 = "tosa.mul"(%266, %266) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%268 = "tosa.reduce_sum"(%267) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%269 = "tosa.mul"(%268, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%270 = "tosa.add"(%269, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%271 = "tosa.rsqrt"(%270) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%272 = "tosa.mul"(%266, %271) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%273 = "tosa.mul"(%272, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%274 = "tosa.add"(%273, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%275 = "tosa.matmul"(%274, %52) : (tensor<1x128x768xf32>, tensor<1x768x2304xf32>) -> tensor<1x128x2304xf32>
%276 = "tosa.reshape"(%275) {new_shape = array<i64: 128, 2304>} : (tensor<1x128x2304xf32>) -> tensor<128x2304xf32>
%277 = "tosa.add"(%55, %276) : (tensor<1x2304xf32>, tensor<128x2304xf32>) -> tensor<128x2304xf32>
%278 = "tosa.reshape"(%277) {new_shape = array<i64: 1, 294912, 1>} : (tensor<128x2304xf32>) -> tensor<1x294912x1xf32>
%279 = "tosa.gather"(%278, %15) : (tensor<1x294912x1xf32>, tensor<1x98304xi32>) -> tensor<1x98304x1xf32>
%280 = "tosa.reshape"(%279) {new_shape = array<i64: 1, 128, 12, 64>} : (tensor<1x98304x1xf32>) -> tensor<1x128x12x64xf32>
%281 = "tosa.transpose"(%280, %16) : (tensor<1x128x12x64xf32>, tensor<4xi64>) -> tensor<1x12x128x64xf32>
%282 = "tosa.transpose"(%281, %17) : (tensor<1x12x128x64xf32>, tensor<4xi32>) -> tensor<1x12x64x128xf32>
%283 = "tosa.reshape"(%281) {new_shape = array<i64: 12, 128, 64>} : (tensor<1x12x128x64xf32>) -> tensor<12x128x64xf32>
%284 = "tosa.reshape"(%282) {new_shape = array<i64: 12, 64, 128>} : (tensor<1x12x64x128xf32>) -> tensor<12x64x128xf32>
%285 = "tosa.matmul"(%283, %284) : (tensor<12x128x64xf32>, tensor<12x64x128xf32>) -> tensor<12x128x128xf32>
%286 = "tosa.reshape"(%285) {new_shape = array<i64: 1, 12, 128, 128>} : (tensor<12x128x128xf32>) -> tensor<1x12x128x128xf32>
%287 = "tosa.mul"(%286, %67) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x1x1x1xf32>) -> tensor<1x12x128x128xf32>
%288 = "tosa.select"(%75, %5, %287) : (tensor<1x1x128x128xi1>, tensor<f32>, tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%289 = "tosa.reduce_max"(%288) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%290 = "tosa.sub"(%288, %289) : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%291 = "tosa.exp"(%290) : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%292 = "tosa.reduce_sum"(%291) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%293 = "tosa.reciprocal"(%292) : (tensor<1x12x128x1xf32>) -> tensor<1x12x128x1xf32>
%294 = "tosa.mul"(%291, %293) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%295 = "tosa.reshape"(%294) {new_shape = array<i64: 12, 128, 128>} : (tensor<1x12x128x128xf32>) -> tensor<12x128x128xf32>
%296 = "tosa.matmul"(%295, %283) : (tensor<12x128x128xf32>, tensor<12x128x64xf32>) -> tensor<12x128x64xf32>
%297 = "tosa.reshape"(%296) {new_shape = array<i64: 1, 12, 128, 64>} : (tensor<12x128x64xf32>) -> tensor<1x12x128x64xf32>
%298 = "tosa.transpose"(%297, %16) : (tensor<1x12x128x64xf32>, tensor<4xi64>) -> tensor<1x128x12x64xf32>
%299 = "tosa.reshape"(%298) {new_shape = array<i64: 1, 128, 768>} : (tensor<1x128x12x64xf32>) -> tensor<1x128x768xf32>
%300 = "tosa.matmul"(%299, %88) : (tensor<1x128x768xf32>, tensor<1x768x768xf32>) -> tensor<1x128x768xf32>
%301 = "tosa.reshape"(%300) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%302 = "tosa.add"(%91, %301) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%303 = "tosa.reshape"(%302) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%304 = "tosa.add"(%303, %263) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%305 = "tosa.reduce_sum"(%304) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%306 = "tosa.mul"(%305, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%307 = "tosa.sub"(%304, %306) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%308 = "tosa.mul"(%307, %307) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%309 = "tosa.reduce_sum"(%308) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%310 = "tosa.mul"(%309, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%311 = "tosa.add"(%310, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%312 = "tosa.rsqrt"(%311) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%313 = "tosa.mul"(%307, %312) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%314 = "tosa.mul"(%313, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%315 = "tosa.add"(%314, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%316 = "tosa.matmul"(%315, %106) : (tensor<1x128x768xf32>, tensor<1x768x3072xf32>) -> tensor<1x128x3072xf32>
%317 = "tosa.reshape"(%316) {new_shape = array<i64: 128, 3072>} : (tensor<1x128x3072xf32>) -> tensor<128x3072xf32>
%318 = "tosa.add"(%109, %317) : (tensor<1x3072xf32>, tensor<128x3072xf32>) -> tensor<128x3072xf32>
%319 = "tosa.reshape"(%318) {new_shape = array<i64: 1, 128, 3072>} : (tensor<128x3072xf32>) -> tensor<1x128x3072xf32>
%320 = "tosa.mul"(%319, %27) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%321 = "tosa.pow"(%319, %26) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%322 = "tosa.mul"(%321, %25) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%323 = "tosa.add"(%319, %322) : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%324 = "tosa.mul"(%323, %24) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%325 = "tosa.tanh"(%324) : (tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%326 = "tosa.add"(%325, %29) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%327 = "tosa.mul"(%320, %326) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%328 = "tosa.matmul"(%327, %120) : (tensor<1x128x3072xf32>, tensor<1x3072x768xf32>) -> tensor<1x128x768xf32>
%329 = "tosa.reshape"(%328) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%330 = "tosa.add"(%91, %329) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%331 = "tosa.reshape"(%330) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%332 = "tosa.add"(%304, %331) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%333 = "tosa.reduce_sum"(%332) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%334 = "tosa.mul"(%333, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%335 = "tosa.sub"(%332, %334) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%336 = "tosa.mul"(%335, %335) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%337 = "tosa.reduce_sum"(%336) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%338 = "tosa.mul"(%337, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%339 = "tosa.add"(%338, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%340 = "tosa.rsqrt"(%339) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%341 = "tosa.mul"(%335, %340) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%342 = "tosa.mul"(%341, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%343 = "tosa.add"(%342, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%344 = "tosa.matmul"(%343, %52) : (tensor<1x128x768xf32>, tensor<1x768x2304xf32>) -> tensor<1x128x2304xf32>
%345 = "tosa.reshape"(%344) {new_shape = array<i64: 128, 2304>} : (tensor<1x128x2304xf32>) -> tensor<128x2304xf32>
%346 = "tosa.add"(%55, %345) : (tensor<1x2304xf32>, tensor<128x2304xf32>) -> tensor<128x2304xf32>
%347 = "tosa.reshape"(%346) {new_shape = array<i64: 1, 294912, 1>} : (tensor<128x2304xf32>) -> tensor<1x294912x1xf32>
%348 = "tosa.gather"(%347, %15) : (tensor<1x294912x1xf32>, tensor<1x98304xi32>) -> tensor<1x98304x1xf32>
%349 = "tosa.reshape"(%348) {new_shape = array<i64: 1, 128, 12, 64>} : (tensor<1x98304x1xf32>) -> tensor<1x128x12x64xf32>
%350 = "tosa.transpose"(%349, %16) : (tensor<1x128x12x64xf32>, tensor<4xi64>) -> tensor<1x12x128x64xf32>
%351 = "tosa.transpose"(%350, %17) : (tensor<1x12x128x64xf32>, tensor<4xi32>) -> tensor<1x12x64x128xf32>
%352 = "tosa.reshape"(%350) {new_shape = array<i64: 12, 128, 64>} : (tensor<1x12x128x64xf32>) -> tensor<12x128x64xf32>
%353 = "tosa.reshape"(%351) {new_shape = array<i64: 12, 64, 128>} : (tensor<1x12x64x128xf32>) -> tensor<12x64x128xf32>
%354 = "tosa.matmul"(%352, %353) : (tensor<12x128x64xf32>, tensor<12x64x128xf32>) -> tensor<12x128x128xf32>
%355 = "tosa.reshape"(%354) {new_shape = array<i64: 1, 12, 128, 128>} : (tensor<12x128x128xf32>) -> tensor<1x12x128x128xf32>
%356 = "tosa.mul"(%355, %67) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x1x1x1xf32>) -> tensor<1x12x128x128xf32>
%357 = "tosa.select"(%75, %5, %356) : (tensor<1x1x128x128xi1>, tensor<f32>, tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%358 = "tosa.reduce_max"(%357) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%359 = "tosa.sub"(%357, %358) : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%360 = "tosa.exp"(%359) : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%361 = "tosa.reduce_sum"(%360) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%362 = "tosa.reciprocal"(%361) : (tensor<1x12x128x1xf32>) -> tensor<1x12x128x1xf32>
%363 = "tosa.mul"(%360, %362) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%364 = "tosa.reshape"(%363) {new_shape = array<i64: 12, 128, 128>} : (tensor<1x12x128x128xf32>) -> tensor<12x128x128xf32>
%365 = "tosa.matmul"(%364, %352) : (tensor<12x128x128xf32>, tensor<12x128x64xf32>) -> tensor<12x128x64xf32>
%366 = "tosa.reshape"(%365) {new_shape = array<i64: 1, 12, 128, 64>} : (tensor<12x128x64xf32>) -> tensor<1x12x128x64xf32>
%367 = "tosa.transpose"(%366, %16) : (tensor<1x12x128x64xf32>, tensor<4xi64>) -> tensor<1x128x12x64xf32>
%368 = "tosa.reshape"(%367) {new_shape = array<i64: 1, 128, 768>} : (tensor<1x128x12x64xf32>) -> tensor<1x128x768xf32>
%369 = "tosa.matmul"(%368, %88) : (tensor<1x128x768xf32>, tensor<1x768x768xf32>) -> tensor<1x128x768xf32>
%370 = "tosa.reshape"(%369) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%371 = "tosa.add"(%91, %370) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%372 = "tosa.reshape"(%371) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%373 = "tosa.add"(%372, %332) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%374 = "tosa.reduce_sum"(%373) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%375 = "tosa.mul"(%374, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%376 = "tosa.sub"(%373, %375) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%377 = "tosa.mul"(%376, %376) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%378 = "tosa.reduce_sum"(%377) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%379 = "tosa.mul"(%378, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%380 = "tosa.add"(%379, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%381 = "tosa.rsqrt"(%380) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%382 = "tosa.mul"(%376, %381) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%383 = "tosa.mul"(%382, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%384 = "tosa.add"(%383, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%385 = "tosa.matmul"(%384, %106) : (tensor<1x128x768xf32>, tensor<1x768x3072xf32>) -> tensor<1x128x3072xf32>
%386 = "tosa.reshape"(%385) {new_shape = array<i64: 128, 3072>} : (tensor<1x128x3072xf32>) -> tensor<128x3072xf32>
%387 = "tosa.add"(%109, %386) : (tensor<1x3072xf32>, tensor<128x3072xf32>) -> tensor<128x3072xf32>
%388 = "tosa.reshape"(%387) {new_shape = array<i64: 1, 128, 3072>} : (tensor<128x3072xf32>) -> tensor<1x128x3072xf32>
%389 = "tosa.mul"(%388, %27) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%390 = "tosa.pow"(%388, %26) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%391 = "tosa.mul"(%390, %25) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%392 = "tosa.add"(%388, %391) : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%393 = "tosa.mul"(%392, %24) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%394 = "tosa.tanh"(%393) : (tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%395 = "tosa.add"(%394, %29) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%396 = "tosa.mul"(%389, %395) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%397 = "tosa.matmul"(%396, %120) : (tensor<1x128x3072xf32>, tensor<1x3072x768xf32>) -> tensor<1x128x768xf32>
%398 = "tosa.reshape"(%397) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%399 = "tosa.add"(%91, %398) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%400 = "tosa.reshape"(%399) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%401 = "tosa.add"(%373, %400) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%402 = "tosa.reduce_sum"(%401) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%403 = "tosa.mul"(%402, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%404 = "tosa.sub"(%401, %403) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%405 = "tosa.mul"(%404, %404) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%406 = "tosa.reduce_sum"(%405) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%407 = "tosa.mul"(%406, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%408 = "tosa.add"(%407, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%409 = "tosa.rsqrt"(%408) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%410 = "tosa.mul"(%404, %409) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%411 = "tosa.mul"(%410, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%412 = "tosa.add"(%411, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%413 = "tosa.matmul"(%412, %52) : (tensor<1x128x768xf32>, tensor<1x768x2304xf32>) -> tensor<1x128x2304xf32>
%414 = "tosa.reshape"(%413) {new_shape = array<i64: 128, 2304>} : (tensor<1x128x2304xf32>) -> tensor<128x2304xf32>
%415 = "tosa.add"(%55, %414) : (tensor<1x2304xf32>, tensor<128x2304xf32>) -> tensor<128x2304xf32>
%416 = "tosa.reshape"(%415) {new_shape = array<i64: 1, 294912, 1>} : (tensor<128x2304xf32>) -> tensor<1x294912x1xf32>
%417 = "tosa.gather"(%416, %15) : (tensor<1x294912x1xf32>, tensor<1x98304xi32>) -> tensor<1x98304x1xf32>
%418 = "tosa.reshape"(%417) {new_shape = array<i64: 1, 128, 12, 64>} : (tensor<1x98304x1xf32>) -> tensor<1x128x12x64xf32>
%419 = "tosa.transpose"(%418, %16) : (tensor<1x128x12x64xf32>, tensor<4xi64>) -> tensor<1x12x128x64xf32>
%420 = "tosa.transpose"(%419, %17) : (tensor<1x12x128x64xf32>, tensor<4xi32>) -> tensor<1x12x64x128xf32>
%421 = "tosa.reshape"(%419) {new_shape = array<i64: 12, 128, 64>} : (tensor<1x12x128x64xf32>) -> tensor<12x128x64xf32>
%422 = "tosa.reshape"(%420) {new_shape = array<i64: 12, 64, 128>} : (tensor<1x12x64x128xf32>) -> tensor<12x64x128xf32>
%423 = "tosa.matmul"(%421, %422) : (tensor<12x128x64xf32>, tensor<12x64x128xf32>) -> tensor<12x128x128xf32>
%424 = "tosa.reshape"(%423) {new_shape = array<i64: 1, 12, 128, 128>} : (tensor<12x128x128xf32>) -> tensor<1x12x128x128xf32>
%425 = "tosa.mul"(%424, %67) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x1x1x1xf32>) -> tensor<1x12x128x128xf32>
%426 = "tosa.select"(%75, %5, %425) : (tensor<1x1x128x128xi1>, tensor<f32>, tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%427 = "tosa.reduce_max"(%426) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%428 = "tosa.sub"(%426, %427) : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%429 = "tosa.exp"(%428) : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x128xf32>
%430 = "tosa.reduce_sum"(%429) {axis = 3 : i64} : (tensor<1x12x128x128xf32>) -> tensor<1x12x128x1xf32>
%431 = "tosa.reciprocal"(%430) : (tensor<1x12x128x1xf32>) -> tensor<1x12x128x1xf32>
%432 = "tosa.mul"(%429, %431) {shift = 0 : i32} : (tensor<1x12x128x128xf32>, tensor<1x12x128x1xf32>) -> tensor<1x12x128x128xf32>
%433 = "tosa.reshape"(%432) {new_shape = array<i64: 12, 128, 128>} : (tensor<1x12x128x128xf32>) -> tensor<12x128x128xf32>
%434 = "tosa.matmul"(%433, %421) : (tensor<12x128x128xf32>, tensor<12x128x64xf32>) -> tensor<12x128x64xf32>
%435 = "tosa.reshape"(%434) {new_shape = array<i64: 1, 12, 128, 64>} : (tensor<12x128x64xf32>) -> tensor<1x12x128x64xf32>
%436 = "tosa.transpose"(%435, %16) : (tensor<1x12x128x64xf32>, tensor<4xi64>) -> tensor<1x128x12x64xf32>
%437 = "tosa.reshape"(%436) {new_shape = array<i64: 1, 128, 768>} : (tensor<1x128x12x64xf32>) -> tensor<1x128x768xf32>
%438 = "tosa.matmul"(%437, %88) : (tensor<1x128x768xf32>, tensor<1x768x768xf32>) -> tensor<1x128x768xf32>
%439 = "tosa.reshape"(%438) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%440 = "tosa.add"(%91, %439) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%441 = "tosa.reshape"(%440) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%442 = "tosa.add"(%441, %401) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%443 = "tosa.reduce_sum"(%442) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%444 = "tosa.mul"(%443, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%445 = "tosa.sub"(%442, %444) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%446 = "tosa.mul"(%445, %445) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%447 = "tosa.reduce_sum"(%446) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%448 = "tosa.mul"(%447, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%449 = "tosa.add"(%448, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%450 = "tosa.rsqrt"(%449) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%451 = "tosa.mul"(%445, %450) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%452 = "tosa.mul"(%451, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%453 = "tosa.add"(%452, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%454 = "tosa.matmul"(%453, %106) : (tensor<1x128x768xf32>, tensor<1x768x3072xf32>) -> tensor<1x128x3072xf32>
%455 = "tosa.reshape"(%454) {new_shape = array<i64: 128, 3072>} : (tensor<1x128x3072xf32>) -> tensor<128x3072xf32>
%456 = "tosa.add"(%109, %455) : (tensor<1x3072xf32>, tensor<128x3072xf32>) -> tensor<128x3072xf32>
%457 = "tosa.reshape"(%456) {new_shape = array<i64: 1, 128, 3072>} : (tensor<128x3072xf32>) -> tensor<1x128x3072xf32>
%458 = "tosa.mul"(%457, %27) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%459 = "tosa.pow"(%457, %26) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%460 = "tosa.mul"(%459, %25) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%461 = "tosa.add"(%457, %460) : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%462 = "tosa.mul"(%461, %24) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%463 = "tosa.tanh"(%462) : (tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%464 = "tosa.add"(%463, %29) : (tensor<1x128x3072xf32>, tensor<1x1x1xf32>) -> tensor<1x128x3072xf32>
%465 = "tosa.mul"(%458, %464) {shift = 0 : i32} : (tensor<1x128x3072xf32>, tensor<1x128x3072xf32>) -> tensor<1x128x3072xf32>
%466 = "tosa.matmul"(%465, %120) : (tensor<1x128x3072xf32>, tensor<1x3072x768xf32>) -> tensor<1x128x768xf32>
%467 = "tosa.reshape"(%466) {new_shape = array<i64: 128, 768>} : (tensor<1x128x768xf32>) -> tensor<128x768xf32>
%468 = "tosa.add"(%91, %467) : (tensor<1x768xf32>, tensor<128x768xf32>) -> tensor<128x768xf32>
%469 = "tosa.reshape"(%468) {new_shape = array<i64: 1, 128, 768>} : (tensor<128x768xf32>) -> tensor<1x128x768xf32>
%470 = "tosa.add"(%442, %469) : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%471 = "tosa.reduce_sum"(%470) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%472 = "tosa.mul"(%471, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%473 = "tosa.sub"(%470, %472) : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%474 = "tosa.mul"(%473, %473) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x768xf32>) -> tensor<1x128x768xf32>
%475 = "tosa.reduce_sum"(%474) {axis = 2 : i64} : (tensor<1x128x768xf32>) -> tensor<1x128x1xf32>
%476 = "tosa.mul"(%475, %40) {shift = 0 : i32} : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%477 = "tosa.add"(%476, %28) : (tensor<1x128x1xf32>, tensor<1x1x1xf32>) -> tensor<1x128x1xf32>
%478 = "tosa.rsqrt"(%477) : (tensor<1x128x1xf32>) -> tensor<1x128x1xf32>
%479 = "tosa.mul"(%473, %478) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x128x1xf32>) -> tensor<1x128x768xf32>
%480 = "tosa.mul"(%479, %49) {shift = 0 : i32} : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%481 = "tosa.add"(%480, %49) : (tensor<1x128x768xf32>, tensor<1x1x768xf32>) -> tensor<1x128x768xf32>
%482 = "tosa.transpose"(%0, %22) : (tensor<2x768xf32>, tensor<2xi32>) -> tensor<768x2xf32>
%483 = "tosa.reshape"(%482) {new_shape = array<i64: 1, 768, 2>} : (tensor<768x2xf32>) -> tensor<1x768x2xf32>
%484 = "tosa.matmul"(%481, %483) : (tensor<1x128x768xf32>, tensor<1x768x2xf32>) -> tensor<1x128x2xf32>
%485 = "tosa.slice"(%484) {size = array<i64: 1, 1, 2>, start = array<i64: 0, 127, 0>} : (tensor<1x128x2xf32>) -> tensor<1x1x2xf32>
%486 = "tosa.cast"(%23) : (tensor<1xi64>) -> tensor<1xi32>
%487 = "tosa.reshape"(%486) {new_shape = array<i64: 1, 1>} : (tensor<1xi32>) -> tensor<1x1xi32>
%488 = "tosa.gather"(%485, %487) : (tensor<1x1x2xf32>, tensor<1x1xi32>) -> tensor<1x1x2xf32>
%489 = "tosa.reshape"(%488) {new_shape = array<i64: 1, 2>} : (tensor<1x1x2xf32>) -> tensor<1x2xf32>
return %489 : tensor<1x2xf32>
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment