Created
June 13, 2026 20:37
-
-
Save ddh0/f1e232417a76a60fd9dfeddc16922613 to your computer and use it in GitHub Desktop.
./build/bin/llama-tensor-debug -lv 4 --device none --no-mmap -b 512 -ub 512 -c 4096 -f ~/datasets/ddh0_imat_calibration_data_v2.txt -m ~/gguf/Gemma-4-31B-StyleTune-Q8_0.gguf
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| warn: LLAMA_ARG_MMAP environment variable is set, but will be overwritten by command line argument --no-mmap | |
| warn: LLAMA_ARG_BATCH environment variable is set, but will be overwritten by command line argument -b | |
| warn: LLAMA_ARG_UBATCH environment variable is set, but will be overwritten by command line argument -ub | |
| 0.00.022.162 W main: parameter override: warmup = false | |
| 0.00.022.549 I system_info: n_threads = 8 (n_threads_batch = 8) / 16 | CUDA : ARCHS = 890 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | | |
| 0.00.022.559 I common_init_result: fitting params to device memory ... | |
| 0.00.022.559 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on) | |
| 0.00.022.804 I common_params_fit_impl: getting device memory data for initial parameters: | |
| 0.00.451.030 I common_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | | |
| 0.00.451.033 I common_memory_breakdown_print: | - Host | 34648 = 32536 + 1520 + 591 | | |
| 0.00.515.644 I common_params_fit_impl: projected to use 34648 MiB of host memory vs. 127916 MiB of total host memory | |
| 0.00.515.646 I common_params_fit_impl: will leave 93268 >= 1024 MiB of system memory, no changes needed | |
| 0.00.515.647 I common_fit_params: successfully fit params to free device memory | |
| 0.00.515.651 I common_fit_params: fitting params to free memory took 0.49 seconds | |
| 0.00.555.022 I llama_model_loader: loaded meta data with 51 key-value pairs and 834 tensors from /home/gguf/Gemma-4-31B-StyleTune-Q8_0.gguf (version GGUF V3 (latest)) | |
| 0.00.555.036 I llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. | |
| 0.00.555.039 I llama_model_loader: - kv 0: general.architecture str = gemma4 | |
| 0.00.555.039 I llama_model_loader: - kv 1: general.type str = model | |
| 0.00.555.040 I llama_model_loader: - kv 2: general.sampling.top_k i32 = 64 | |
| 0.00.555.043 I llama_model_loader: - kv 3: general.sampling.top_p f32 = 0.950000 | |
| 0.00.555.044 I llama_model_loader: - kv 4: general.sampling.temp f32 = 1.000000 | |
| 0.00.555.044 I llama_model_loader: - kv 5: general.name str = Gemma-4-31B-StyleTune | |
| 0.00.555.044 I llama_model_loader: - kv 6: general.finetune str = StyleTune | |
| 0.00.555.045 I llama_model_loader: - kv 7: general.basename str = Gemma-4 | |
| 0.00.555.045 I llama_model_loader: - kv 8: general.size_label str = 31B | |
| 0.00.555.045 I llama_model_loader: - kv 9: general.license str = apache-2.0 | |
| 0.00.555.046 I llama_model_loader: - kv 10: general.base_model.count u32 = 1 | |
| 0.00.555.046 I llama_model_loader: - kv 11: general.base_model.0.name str = Gemma 4 31B It | |
| 0.00.555.054 I llama_model_loader: - kv 12: general.base_model.0.organization str = Google | |
| 0.00.555.056 I llama_model_loader: - kv 13: general.base_model.0.repo_url str = https://huggingface.co/google/gemma-4... | |
| 0.00.555.064 I llama_model_loader: - kv 14: general.tags arr[str,8] = ["gemma4", "conversational", "instruc... | |
| 0.00.555.064 I llama_model_loader: - kv 15: general.languages arr[str,1] = ["en"] | |
| 0.00.555.065 I llama_model_loader: - kv 16: gemma4.block_count u32 = 60 | |
| 0.00.555.065 I llama_model_loader: - kv 17: gemma4.context_length u32 = 262144 | |
| 0.00.555.066 I llama_model_loader: - kv 18: gemma4.embedding_length u32 = 5376 | |
| 0.00.555.066 I llama_model_loader: - kv 19: gemma4.feed_forward_length u32 = 21504 | |
| 0.00.555.066 I llama_model_loader: - kv 20: gemma4.attention.head_count u32 = 32 | |
| 0.00.555.069 I llama_model_loader: - kv 21: gemma4.attention.head_count_kv arr[i32,60] = [16, 16, 16, 16, 16, 4, 16, 16, 16, 1... | |
| 0.00.555.070 I llama_model_loader: - kv 22: gemma4.rope.freq_base f32 = 1000000.000000 | |
| 0.00.555.071 I llama_model_loader: - kv 23: gemma4.rope.freq_base_swa f32 = 10000.000000 | |
| 0.00.555.071 I llama_model_loader: - kv 24: gemma4.attention.layer_norm_rms_epsilon f32 = 0.000001 | |
| 0.00.555.072 I llama_model_loader: - kv 25: gemma4.attention.key_length u32 = 512 | |
| 0.00.555.072 I llama_model_loader: - kv 26: gemma4.attention.value_length u32 = 512 | |
| 0.00.555.073 I llama_model_loader: - kv 27: gemma4.final_logit_softcapping f32 = 30.000000 | |
| 0.00.555.073 I llama_model_loader: - kv 28: gemma4.attention.sliding_window u32 = 1024 | |
| 0.00.555.074 I llama_model_loader: - kv 29: gemma4.attention.shared_kv_layers u32 = 0 | |
| 0.00.555.074 I llama_model_loader: - kv 30: gemma4.embedding_length_per_layer_input u32 = 0 | |
| 0.00.555.076 I llama_model_loader: - kv 31: gemma4.attention.sliding_window_pattern arr[bool,60] = [true, true, true, true, true, false,... | |
| 0.00.555.077 I llama_model_loader: - kv 32: gemma4.attention.key_length_swa u32 = 256 | |
| 0.00.555.077 I llama_model_loader: - kv 33: gemma4.attention.value_length_swa u32 = 256 | |
| 0.00.555.077 I llama_model_loader: - kv 34: gemma4.rope.dimension_count u32 = 512 | |
| 0.00.555.078 I llama_model_loader: - kv 35: gemma4.rope.dimension_count_swa u32 = 256 | |
| 0.00.555.078 I llama_model_loader: - kv 36: tokenizer.ggml.model str = gemma4 | |
| 0.00.571.645 I llama_model_loader: - kv 37: tokenizer.ggml.tokens arr[str,262144] = ["<pad>", "<eos>", "<bos>", "<unk>", ... | |
| 0.00.604.062 I llama_model_loader: - kv 38: tokenizer.ggml.scores arr[f32,262144] = [-1000.000000, -1000.000000, -1000.00... | |
| 0.00.608.961 I llama_model_loader: - kv 39: tokenizer.ggml.token_type arr[i32,262144] = [3, 1, 3, 3, 3, 1, 1, 1, 1, 1, 1, 1, ... | |
| 0.00.640.889 I llama_model_loader: - kv 40: tokenizer.ggml.merges arr[str,514906] = ["\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n \n", ... | |
| 0.00.640.892 I llama_model_loader: - kv 41: tokenizer.ggml.bos_token_id u32 = 2 | |
| 0.00.640.892 I llama_model_loader: - kv 42: tokenizer.ggml.eos_token_id u32 = 106 | |
| 0.00.640.893 I llama_model_loader: - kv 43: tokenizer.ggml.unknown_token_id u32 = 3 | |
| 0.00.640.893 I llama_model_loader: - kv 44: tokenizer.ggml.padding_token_id u32 = 0 | |
| 0.00.640.894 I llama_model_loader: - kv 45: tokenizer.ggml.mask_token_id u32 = 4 | |
| 0.00.640.896 I llama_model_loader: - kv 46: tokenizer.chat_template str = {%- macro format_parameters(propertie... | |
| 0.00.640.896 I llama_model_loader: - kv 47: tokenizer.ggml.add_space_prefix bool = false | |
| 0.00.640.897 I llama_model_loader: - kv 48: tokenizer.ggml.add_bos_token bool = true | |
| 0.00.640.897 I llama_model_loader: - kv 49: general.quantization_version u32 = 2 | |
| 0.00.640.897 I llama_model_loader: - kv 50: general.file_type u32 = 7 | |
| 0.00.640.898 I llama_model_loader: - type f32: 422 tensors | |
| 0.00.640.899 I llama_model_loader: - type q8_0: 412 tensors | |
| 0.00.640.900 I print_info: file format = GGUF V3 (latest) | |
| 0.00.640.900 I print_info: file type = Q8_0 | |
| 0.00.640.902 I print_info: file size = 31.77 GiB (8.50 BPW) | |
| 0.00.780.835 I load: 0 unused tokens | |
| 0.00.787.912 W load: control-looking token: 212 '</s>' was not control-type; this is probably a bug in the model. its type will be overridden | |
| 0.00.788.156 W load: control-looking token: 50 '<|tool_response>' was not control-type; this is probably a bug in the model. its type will be overridden | |
| 0.00.791.315 W load: control-looking token: 1 '<eos>' was not control-type; this is probably a bug in the model. its type will be overridden | |
| 0.00.800.271 I load: printing all EOG tokens: | |
| 0.00.800.273 I load: - 1 ('<eos>') | |
| 0.00.800.273 I load: - 50 ('<|tool_response>') | |
| 0.00.800.273 I load: - 106 ('<turn|>') | |
| 0.00.800.274 I load: - 212 ('</s>') | |
| 0.00.800.275 W load: special_eog_ids contains '<|tool_response>', removing '</s>' token from EOG list | |
| 0.00.800.431 I load: special tokens cache size = 24 | |
| 0.00.816.863 I load: token to piece cache size = 1.9445 MB | |
| 0.00.816.872 I print_info: arch = gemma4 | |
| 0.00.816.873 I print_info: vocab_only = 0 | |
| 0.00.816.873 I print_info: no_alloc = 0 | |
| 0.00.816.873 I print_info: n_ctx_train = 262144 | |
| 0.00.816.874 I print_info: n_embd_inp = 5376 | |
| 0.00.816.874 I print_info: n_embd = 5376 | |
| 0.00.816.874 I print_info: n_embd_out = 5376 | |
| 0.00.816.874 I print_info: n_layer = 60 | |
| 0.00.816.874 I print_info: n_layer_all = 60 | |
| 0.00.816.880 I print_info: n_head = 32 | |
| 0.00.816.884 I print_info: n_head_kv = [16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4, 16, 16, 16, 16, 16, 4] | |
| 0.00.816.884 I print_info: n_rot = 512 | |
| 0.00.816.884 I print_info: n_swa = 1024 | |
| 0.00.816.885 I print_info: is_swa_any = 1 | |
| 0.00.816.885 I print_info: n_embd_head_k = 512 | |
| 0.00.816.885 I print_info: n_embd_head_v = 512 | |
| 0.00.816.887 I print_info: n_gqa = [2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8, 2, 2, 2, 2, 2, 8] | |
| 0.00.816.890 I print_info: n_embd_k_gqa = [4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048] | |
| 0.00.816.892 I print_info: n_embd_v_gqa = [4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048, 4096, 4096, 4096, 4096, 4096, 2048] | |
| 0.00.816.893 I print_info: f_norm_eps = 0.0e+00 | |
| 0.00.816.894 I print_info: f_norm_rms_eps = 1.0e-06 | |
| 0.00.816.894 I print_info: f_clamp_kqv = 0.0e+00 | |
| 0.00.816.894 I print_info: f_max_alibi_bias = 0.0e+00 | |
| 0.00.816.894 I print_info: f_logit_scale = 0.0e+00 | |
| 0.00.816.894 I print_info: f_attn_scale = 1.0e+00 | |
| 0.00.816.895 I print_info: f_attn_value_scale = 0.0000 | |
| 0.00.816.896 I print_info: n_ff = 21504 | |
| 0.00.816.896 I print_info: n_expert = 0 | |
| 0.00.816.896 I print_info: n_expert_used = 0 | |
| 0.00.816.896 I print_info: n_expert_groups = 0 | |
| 0.00.816.896 I print_info: n_group_used = 0 | |
| 0.00.816.896 I print_info: causal attn = 1 | |
| 0.00.816.896 I print_info: pooling type = -1 | |
| 0.00.816.896 I print_info: rope type = 2 | |
| 0.00.816.897 I print_info: rope scaling = linear | |
| 0.00.816.897 I print_info: freq_base_train = 1000000.0 | |
| 0.00.816.898 I print_info: freq_scale_train = 1 | |
| 0.00.816.898 I print_info: freq_base_swa = 10000.0 | |
| 0.00.816.898 I print_info: freq_scale_swa = 1 | |
| 0.00.816.898 I print_info: n_embd_head_k_swa = 256 | |
| 0.00.816.898 I print_info: n_embd_head_v_swa = 256 | |
| 0.00.816.899 I print_info: n_rot_swa = 256 | |
| 0.00.816.899 I print_info: n_ctx_orig_yarn = 262144 | |
| 0.00.816.899 I print_info: rope_yarn_log_mul = 0.0000 | |
| 0.00.816.899 I print_info: rope_finetuned = unknown | |
| 0.00.816.899 I print_info: model type = 31B | |
| 0.00.816.900 I print_info: model params = 32.11 B | |
| 0.00.816.900 I print_info: general.name = Gemma-4-31B-StyleTune | |
| 0.00.816.901 I print_info: vocab type = BPE | |
| 0.00.816.901 I print_info: n_vocab = 262144 | |
| 0.00.816.901 I print_info: n_merges = 514906 | |
| 0.00.816.902 I print_info: BOS token = 2 '<bos>' | |
| 0.00.816.902 I print_info: EOS token = 106 '<turn|>' | |
| 0.00.816.902 I print_info: UNK token = 3 '<unk>' | |
| 0.00.816.902 I print_info: PAD token = 0 '<pad>' | |
| 0.00.816.903 I print_info: MASK token = 4 '<mask>' | |
| 0.00.816.903 I print_info: LF token = 107 ' | |
| ' | |
| 0.00.816.903 I print_info: EOG token = 1 '<eos>' | |
| 0.00.816.903 I print_info: EOG token = 50 '<|tool_response>' | |
| 0.00.816.903 I print_info: EOG token = 106 '<turn|>' | |
| 0.00.816.903 I print_info: max token length = 93 | |
| 0.00.816.904 I load_tensors: loading model tensors, this can take a while... (mmap = false, direct_io = false) | |
| 0.00.825.203 I load_tensors: offloading output layer to GPU | |
| 0.00.825.205 I load_tensors: offloading 59 repeating layers to GPU | |
| 0.00.825.205 I load_tensors: offloaded 61/61 layers to GPU | |
| 0.00.825.206 I load_tensors: CPU model buffer size = 32536.71 MiB | |
| .............................................................................................. | |
| 0.18.345.379 I common_init_result: added <eos> logit bias = -inf | |
| 0.18.345.384 I common_init_result: added <|tool_response> logit bias = -inf | |
| 0.18.345.385 I common_init_result: added <turn|> logit bias = -inf | |
| 0.18.347.428 I llama_context: constructing llama_context | |
| 0.18.347.431 I llama_context: n_seq_max = 1 | |
| 0.18.347.432 I llama_context: n_ctx = 4096 | |
| 0.18.347.432 I llama_context: n_ctx_seq = 4096 | |
| 0.18.347.432 I llama_context: n_batch = 512 | |
| 0.18.347.432 I llama_context: n_ubatch = 512 | |
| 0.18.347.432 I llama_context: causal_attn = 1 | |
| 0.18.347.433 I llama_context: flash_attn = auto | |
| 0.18.347.433 I llama_context: kv_unified = false | |
| 0.18.347.435 I llama_context: freq_base = 1000000.0 | |
| 0.18.347.436 I llama_context: freq_scale = 1 | |
| 0.18.347.436 I llama_context: n_rs_seq = 0 | |
| 0.18.347.436 I llama_context: n_outputs_max = 512 | |
| 0.18.347.437 W llama_context: n_ctx_seq (4096) < n_ctx_train (262144) -- the full capacity of the model will not be utilized | |
| 0.18.351.614 I llama_context: CPU output buffer size = 1.00 MiB | |
| 0.18.351.616 I llama_kv_cache_iswa: creating non-SWA KV cache, size = 4096 cells | |
| 0.18.351.660 I llama_kv_cache: CPU KV buffer size = 320.00 MiB | |
| 0.18.450.407 I llama_kv_cache: size = 320.00 MiB ( 4096 cells, 10 layers, 1/1 seqs), K (f16): 160.00 MiB, V (f16): 160.00 MiB | |
| 0.18.450.411 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 512 | |
| 0.18.450.411 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 512 | |
| 0.18.450.413 I llama_kv_cache_iswa: creating SWA KV cache, size = 1536 cells | |
| 0.18.450.488 I llama_kv_cache: CPU KV buffer size = 1200.00 MiB | |
| 0.18.820.305 I llama_kv_cache: size = 1200.00 MiB ( 1536 cells, 50 layers, 1/1 seqs), K (f16): 600.00 MiB, V (f16): 600.00 MiB | |
| 0.18.820.309 I llama_kv_cache: attn_rot_k = 0, n_embd_head_k_all = 256 | |
| 0.18.820.310 I llama_kv_cache: attn_rot_v = 0, n_embd_head_k_all = 256 | |
| 0.18.820.321 I sched_reserve: reserving ... | |
| 0.18.821.306 I sched_reserve: Flash Attention was auto, set to enabled | |
| 0.18.821.307 I sched_reserve: resolving fused Gated Delta Net support: | |
| 0.18.821.627 I sched_reserve: fused Gated Delta Net (autoregressive) enabled | |
| 0.18.821.936 I sched_reserve: fused Gated Delta Net (chunked) enabled | |
| 0.18.823.310 I sched_reserve: CPU compute buffer size = 591.52 MiB | |
| 0.18.823.312 I sched_reserve: graph nodes = 2459 | |
| 0.18.823.312 I sched_reserve: graph splits = 1 | |
| 0.18.823.313 I sched_reserve: reserve took 2.99 ms, sched copies = 1 | |
| 0.18.823.384 W main: --output-dir parameter not set; observed tensors will not be saved! | |
| 0.18.823.388 I run_debug: tokenizing prompt ... | |
| 0.18.886.993 I run_debug: n_prompt_tokens = 49556 | |
| 0.18.887.003 I run_debug: decoding 512 tokens for debugging ... this may take a while ... | |
| 0.18.912.103 I 00001: f32: embd [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.48, avg = 0.00, max = 0.47, var = 0.00 | |
| 0.18.931.984 I 00002: f32: inp_scaled [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.94, avg = 0.00, max = 34.23, var = 2.50 | |
| 0.18.952.960 I 00003: f32: norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.81, avg = 0.00, max = 26.44, var = 1.00 | |
| 0.18.973.069 I 00004: f32: attn_norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -807.05, avg = -0.07, max = 615.73, var = 67.60 | |
| 0.19.056.844 I 00005: f32: Qcur-0 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -390.73, avg = 0.50, max = 389.31, var = 387.09 | |
| 0.19.087.086 I 00006: f32: Qcur-0 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -390.73, avg = 0.50, max = 389.31, var = 387.09 | |
| 0.19.121.005 I 00007: f32: norm-0 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.56, avg = 0.02, max = 10.12, var = 1.00 | |
| 0.19.151.523 I 00008: f32: Qcur_normed-0 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.79, avg = 0.02, max = 10.35, var = 1.05 | |
| 0.19.182.603 I 00009: f32: Qcur_pos-0 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.39, avg = 0.01, max = 10.42, var = 1.05 | |
| 0.19.217.100 I 00010: f32: Kcur-0 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -336.88, avg = 0.42, max = 503.34, var = 722.01 | |
| 0.19.232.223 I 00011: f32: Kcur-0 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -336.88, avg = 0.42, max = 503.34, var = 722.01 | |
| 0.19.247.910 I 00012: f32: norm-0 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.51, avg = 0.01, max = 13.77, var = 1.00 | |
| 0.19.263.478 I 00013: f32: Kcur_normed-0 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = 0.00, max = 1.68, var = 0.01 | |
| 0.19.279.338 I 00014: f32: Kcur_pos-0 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.71, var = 0.01 | |
| 0.19.320.261 I 00015: f32: Vcur-0 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -695.31, avg = -0.10, max = 1036.41, var = 268.39 | |
| 0.19.335.389 I 00016: f32: Vcur-0 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -695.31, avg = -0.10, max = 1036.41, var = 268.39 | |
| 0.19.351.364 I 00017: f32: Vcur_normed-0 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.92, avg = -0.01, max = 15.94, var = 1.00 | |
| 0.19.366.704 I 00018: f32: Kcur_pos-0 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.71, var = 0.01 | |
| 0.19.410.636 I 00019: f16: cache_k_l0 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.71, var = 0.00 | |
| 0.19.425.633 I 00020: f32: Vcur_normed-0 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.92, avg = -0.01, max = 15.94, var = 1.00 | |
| 0.19.469.453 I 00021: f16: cache_v_l0 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.92, avg = -0.00, max = 15.94, var = 0.33 | |
| 0.19.499.580 I 00022: f32: Qcur_pos-0 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.39, avg = 0.01, max = 10.42, var = 1.05 | |
| 0.19.529.791 I 00023: f32: Qcur_pos-0 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.39, avg = 0.01, max = 10.42, var = 1.05 | |
| 0.19.544.462 I 00024: f16: cache_k_l0 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.71, var = 0.01 | |
| 0.19.559.085 I 00025: f16: cache_k_l0 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.71, var = 0.01 | |
| 0.19.573.638 I 00026: f16: cache_v_l0 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.92, avg = -0.01, max = 15.94, var = 1.00 | |
| 0.19.588.186 I 00027: f16: cache_v_l0 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.92, avg = -0.01, max = 15.94, var = 1.00 | |
| 0.19.630.271 I 00028: f32: __fattn__-0 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.60, avg = 0.00, max = 15.91, var = 0.66 | |
| 0.19.660.490 I 00029: f32: kqv_out-0 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.60, avg = 0.00, max = 15.91, var = 0.66 | |
| 0.19.733.375 I 00030: f32: node_29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -91.90, avg = -0.01, max = 117.75, var = 41.40 | |
| 0.19.753.590 I 00031: f32: norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.18, avg = 0.00, max = 32.44, var = 1.00 | |
| 0.19.773.738 I 00032: f32: attn_post_norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -393.21, avg = 0.15, max = 701.55, var = 95.10 | |
| 0.19.794.006 I 00033: f32: attn_out-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -394.78, avg = 0.15, max = 702.21, var = 97.40 | |
| 0.19.814.602 I 00034: f32: norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.37, avg = 0.02, max = 46.74, var = 1.00 | |
| 0.19.834.705 I 00035: f32: ffn_norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.07, avg = -0.00, max = 28.51, var = 0.49 | |
| 0.20.012.586 I 00036: f32: ffn_gate-0 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.48, avg = -3.01, max = 15.43, var = 1.72 | |
| 0.20.205.438 I 00037: f32: ffn_up-0 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.81, avg = -0.00, max = 30.35, var = 1.75 | |
| 0.20.287.701 I 00038: f32: ffn_geglu-0 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -169.11, avg = 0.00, max = 447.80, var = 0.29 | |
| 0.20.414.854 I 00039: f32: ffn_out-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -88.23, avg = 0.00, max = 107.16, var = 3.56 | |
| 0.20.435.102 I 00040: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.82, avg = 0.00, max = 9.78, var = 1.00 | |
| 0.20.455.087 I 00041: f32: ffn_post_norm-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -46.87, avg = -0.00, max = 51.87, var = 3.42 | |
| 0.20.475.201 I 00042: f32: node_41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -420.72, avg = 0.15, max = 738.31, var = 107.84 | |
| 0.20.496.312 I 00043: f32: l_out-0 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.59, avg = 0.01, max = 65.97, var = 0.86 | |
| 0.20.516.729 I 00044: f32: norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.07, avg = 0.01, max = 46.51, var = 1.00 | |
| 0.20.536.668 I 00045: f32: attn_norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -65.02, avg = 0.06, max = 221.03, var = 14.19 | |
| 0.20.603.726 I 00046: f32: Qcur-1 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -133.41, avg = -0.02, max = 154.13, var = 78.71 | |
| 0.20.634.182 I 00047: f32: Qcur-1 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -133.41, avg = -0.02, max = 154.13, var = 78.71 | |
| 0.20.665.105 I 00048: f32: norm-1 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.51, avg = -0.00, max = 10.22, var = 1.00 | |
| 0.20.695.602 I 00049: f32: Qcur_normed-1 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.98, avg = -0.00, max = 9.70, var = 0.90 | |
| 0.20.726.308 I 00050: f32: Qcur_pos-1 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.16, avg = -0.00, max = 9.67, var = 0.90 | |
| 0.20.760.094 I 00051: f32: Kcur-1 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -406.22, avg = -0.03, max = 344.99, var = 206.02 | |
| 0.20.775.225 I 00052: f32: Kcur-1 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -406.22, avg = -0.03, max = 344.99, var = 206.02 | |
| 0.20.790.904 I 00053: f32: norm-1 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.68, avg = -0.01, max = 13.15, var = 1.00 | |
| 0.20.806.309 I 00054: f32: Kcur_normed-1 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.80, avg = -0.00, max = 1.73, var = 0.02 | |
| 0.20.821.862 I 00055: f32: Kcur_pos-1 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.93, avg = -0.00, max = 1.72, var = 0.02 | |
| 0.20.855.661 I 00056: f32: Vcur-1 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -849.09, avg = 0.15, max = 1036.50, var = 96.47 | |
| 0.20.870.789 I 00057: f32: Vcur-1 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -849.09, avg = 0.15, max = 1036.50, var = 96.47 | |
| 0.20.886.460 I 00058: f32: Vcur_normed-1 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.89, avg = 0.01, max = 15.92, var = 1.00 | |
| 0.20.901.621 I 00059: f32: Kcur_pos-1 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.93, avg = -0.00, max = 1.72, var = 0.02 | |
| 0.20.945.605 I 00060: f16: cache_k_l1 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.93, avg = -0.00, max = 1.72, var = 0.01 | |
| 0.20.960.489 I 00061: f32: Vcur_normed-1 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.89, avg = 0.01, max = 15.92, var = 1.00 | |
| 0.21.004.367 I 00062: f16: cache_v_l1 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.89, avg = 0.00, max = 15.92, var = 0.33 | |
| 0.21.034.422 I 00063: f32: Qcur_pos-1 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.16, avg = -0.00, max = 9.67, var = 0.90 | |
| 0.21.064.641 I 00064: f32: Qcur_pos-1 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.16, avg = -0.00, max = 9.67, var = 0.90 | |
| 0.21.079.306 I 00065: f16: cache_k_l1 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.93, avg = -0.00, max = 1.72, var = 0.02 | |
| 0.21.093.941 I 00066: f16: cache_k_l1 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.93, avg = -0.00, max = 1.72, var = 0.02 | |
| 0.21.108.514 I 00067: f16: cache_v_l1 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.89, avg = 0.01, max = 15.92, var = 1.00 | |
| 0.21.123.096 I 00068: f16: cache_v_l1 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.89, avg = 0.01, max = 15.92, var = 1.00 | |
| 0.21.163.685 I 00069: f32: __fattn__-1 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.81, avg = 0.01, max = 15.90, var = 0.57 | |
| 0.21.193.970 I 00070: f32: kqv_out-1 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.81, avg = 0.01, max = 15.90, var = 0.57 | |
| 0.21.250.841 I 00071: f32: node_70 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.73, avg = -0.02, max = 176.64, var = 15.02 | |
| 0.21.271.376 I 00072: f32: norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.44, avg = -0.00, max = 45.51, var = 1.00 | |
| 0.21.291.287 I 00073: f32: attn_post_norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.26, avg = 0.03, max = 123.73, var = 3.37 | |
| 0.21.311.593 I 00074: f32: attn_out-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -69.18, avg = 0.04, max = 183.96, var = 6.74 | |
| 0.21.331.718 I 00075: f32: norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.17, avg = 0.02, max = 54.02, var = 1.00 | |
| 0.21.351.647 I 00076: f32: ffn_norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.65, avg = 0.01, max = 53.57, var = 0.48 | |
| 0.21.527.609 I 00077: f32: ffn_gate-1 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.19, avg = -1.14, max = 31.69, var = 1.37 | |
| 0.21.703.765 I 00078: f32: ffn_up-1 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.74, avg = 0.01, max = 41.91, var = 1.14 | |
| 0.21.784.787 I 00079: f32: ffn_geglu-1 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -166.53, avg = 0.00, max = 1327.92, var = 0.87 | |
| 0.21.902.129 I 00080: f32: ffn_out-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -330.12, avg = -0.00, max = 547.08, var = 9.83 | |
| 0.21.922.624 I 00081: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.70, avg = 0.00, max = 53.40, var = 1.00 | |
| 0.21.942.341 I 00082: f32: ffn_post_norm-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -626.31, avg = 0.11, max = 1615.39, var = 332.63 | |
| 0.21.962.599 I 00083: f32: node_82 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -627.91, avg = 0.16, max = 1645.32, var = 383.61 | |
| 0.21.982.625 I 00084: f32: l_out-1 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.08, avg = 0.01, max = 107.65, var = 1.64 | |
| 0.22.002.626 I 00085: f32: norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.68, avg = 0.01, max = 62.16, var = 1.00 | |
| 0.22.022.550 I 00086: f32: attn_norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.53, avg = 0.01, max = 353.52, var = 1.59 | |
| 0.22.089.970 I 00087: f32: Qcur-2 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -63.99, avg = -0.04, max = 63.49, var = 13.86 | |
| 0.22.120.190 I 00088: f32: Qcur-2 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -63.99, avg = -0.04, max = 63.49, var = 13.86 | |
| 0.22.151.320 I 00089: f32: norm-2 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.57, avg = -0.01, max = 11.03, var = 1.00 | |
| 0.22.181.853 I 00090: f32: Qcur_normed-2 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.52, avg = -0.01, max = 10.99, var = 0.99 | |
| 0.22.212.432 I 00091: f32: Qcur_pos-2 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.32, avg = -0.01, max = 12.01, var = 0.99 | |
| 0.22.246.232 I 00092: f32: Kcur-2 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -70.41, avg = -0.19, max = 94.77, var = 30.04 | |
| 0.22.261.367 I 00093: f32: Kcur-2 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -70.41, avg = -0.19, max = 94.77, var = 30.04 | |
| 0.22.277.050 I 00094: f32: norm-2 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.83, avg = -0.03, max = 11.12, var = 1.00 | |
| 0.22.292.436 I 00095: f32: Kcur_normed-2 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.99, avg = -0.00, max = 1.40, var = 0.02 | |
| 0.22.307.993 I 00096: f32: Kcur_pos-2 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.99, avg = -0.00, max = 1.40, var = 0.02 | |
| 0.22.341.779 I 00097: f32: Vcur-2 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -220.18, avg = 0.01, max = 263.04, var = 8.46 | |
| 0.22.356.910 I 00098: f32: Vcur-2 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -220.18, avg = 0.01, max = 263.04, var = 8.46 | |
| 0.22.372.608 I 00099: f32: Vcur_normed-2 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.92, avg = 0.00, max = 15.94, var = 1.00 | |
| 0.22.387.817 I 00100: f32: Kcur_pos-2 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.99, avg = -0.00, max = 1.40, var = 0.02 | |
| 0.22.431.812 I 00101: f16: cache_k_l2 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.99, avg = -0.00, max = 1.40, var = 0.01 | |
| 0.22.446.863 I 00102: f32: Vcur_normed-2 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.92, avg = 0.00, max = 15.94, var = 1.00 | |
| 0.22.490.739 I 00103: f16: cache_v_l2 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.92, avg = 0.00, max = 15.95, var = 0.33 | |
| 0.22.520.746 I 00104: f32: Qcur_pos-2 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.32, avg = -0.01, max = 12.01, var = 0.99 | |
| 0.22.550.970 I 00105: f32: Qcur_pos-2 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.32, avg = -0.01, max = 12.01, var = 0.99 | |
| 0.22.565.613 I 00106: f16: cache_k_l2 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.99, avg = -0.00, max = 1.40, var = 0.02 | |
| 0.22.580.267 I 00107: f16: cache_k_l2 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.99, avg = -0.00, max = 1.40, var = 0.02 | |
| 0.22.594.841 I 00108: f16: cache_v_l2 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.92, avg = 0.00, max = 15.95, var = 1.00 | |
| 0.22.609.403 I 00109: f16: cache_v_l2 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.92, avg = 0.00, max = 15.95, var = 1.00 | |
| 0.22.650.238 I 00110: f32: __fattn__-2 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = 0.00, max = 15.95, var = 0.63 | |
| 0.22.680.495 I 00111: f32: kqv_out-2 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = 0.00, max = 15.95, var = 0.63 | |
| 0.22.737.551 I 00112: f32: node_111 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -86.70, avg = 0.04, max = 136.04, var = 17.44 | |
| 0.22.757.976 I 00113: f32: norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 33.66, var = 1.00 | |
| 0.22.778.038 I 00114: f32: attn_post_norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -3.39, avg = 0.00, max = 5.23, var = 0.01 | |
| 0.22.797.870 I 00115: f32: attn_out-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -43.95, avg = 0.01, max = 106.44, var = 1.80 | |
| 0.22.817.961 I 00116: f32: norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.20, avg = 0.01, max = 61.28, var = 1.00 | |
| 0.22.837.845 I 00117: f32: ffn_norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.49, avg = 0.01, max = 23.40, var = 0.66 | |
| 0.23.013.946 I 00118: f32: ffn_gate-2 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.25, avg = -2.74, max = 19.45, var = 1.87 | |
| 0.23.190.112 I 00119: f32: ffn_up-2 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.78, avg = 0.00, max = 56.53, var = 1.15 | |
| 0.23.271.206 I 00120: f32: ffn_geglu-2 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -229.82, avg = -0.00, max = 461.56, var = 0.52 | |
| 0.23.388.313 I 00121: f32: ffn_out-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -110.61, avg = -0.01, max = 154.97, var = 8.95 | |
| 0.23.408.719 I 00122: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.98, avg = -0.00, max = 9.28, var = 1.00 | |
| 0.23.428.333 I 00123: f32: ffn_post_norm-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.75, avg = 0.00, max = 2.04, var = 0.00 | |
| 0.23.448.137 I 00124: f32: node_123 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -45.41, avg = 0.01, max = 106.44, var = 1.85 | |
| 0.23.468.343 I 00125: f32: l_out-2 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -45.05, avg = 0.01, max = 105.61, var = 1.82 | |
| 0.23.488.355 I 00126: f32: norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.45, avg = 0.01, max = 61.44, var = 1.00 | |
| 0.23.508.236 I 00127: f32: attn_norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.24, avg = 0.02, max = 629.75, var = 1.37 | |
| 0.23.575.690 I 00128: f32: Qcur-3 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -83.88, avg = -0.05, max = 86.94, var = 16.10 | |
| 0.23.605.922 I 00129: f32: Qcur-3 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -83.88, avg = -0.05, max = 86.94, var = 16.10 | |
| 0.23.637.046 I 00130: f32: norm-3 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.17, avg = -0.01, max = 12.38, var = 1.00 | |
| 0.23.667.559 I 00131: f32: Qcur_normed-3 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.27, avg = -0.01, max = 12.47, var = 1.02 | |
| 0.23.698.444 I 00132: f32: Qcur_pos-3 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.87, avg = -0.00, max = 12.74, var = 1.02 | |
| 0.23.732.516 I 00133: f32: Kcur-3 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -84.27, avg = 0.02, max = 85.08, var = 27.77 | |
| 0.23.747.646 I 00134: f32: Kcur-3 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -84.27, avg = 0.02, max = 85.08, var = 27.77 | |
| 0.23.763.314 I 00135: f32: norm-3 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.12, avg = 0.00, max = 8.20, var = 1.00 | |
| 0.23.778.716 I 00136: f32: Kcur_normed-3 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.01, avg = 0.00, max = 1.02, var = 0.02 | |
| 0.23.794.281 I 00137: f32: Kcur_pos-3 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.01, avg = 0.00, max = 1.02, var = 0.02 | |
| 0.23.828.081 I 00138: f32: Vcur-3 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -199.40, avg = -0.00, max = 224.30, var = 7.91 | |
| 0.23.843.211 I 00139: f32: Vcur-3 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -199.40, avg = -0.00, max = 224.30, var = 7.91 | |
| 0.23.858.835 I 00140: f32: Vcur_normed-3 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.93, avg = 0.00, max = 15.60, var = 1.00 | |
| 0.23.873.995 I 00141: f32: Kcur_pos-3 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.01, avg = 0.00, max = 1.02, var = 0.02 | |
| 0.23.917.983 I 00142: f16: cache_k_l3 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.01, avg = 0.00, max = 1.02, var = 0.01 | |
| 0.23.932.857 I 00143: f32: Vcur_normed-3 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.93, avg = 0.00, max = 15.60, var = 1.00 | |
| 0.23.976.696 I 00144: f16: cache_v_l3 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.94, avg = 0.00, max = 15.60, var = 0.33 | |
| 0.24.006.643 I 00145: f32: Qcur_pos-3 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.87, avg = -0.00, max = 12.74, var = 1.02 | |
| 0.24.036.863 I 00146: f32: Qcur_pos-3 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.87, avg = -0.00, max = 12.74, var = 1.02 | |
| 0.24.051.496 I 00147: f16: cache_k_l3 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.01, avg = 0.00, max = 1.02, var = 0.02 | |
| 0.24.066.107 I 00148: f16: cache_k_l3 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.01, avg = 0.00, max = 1.02, var = 0.02 | |
| 0.24.080.678 I 00149: f16: cache_v_l3 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.94, avg = 0.00, max = 15.60, var = 1.00 | |
| 0.24.095.233 I 00150: f16: cache_v_l3 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.94, avg = 0.00, max = 15.60, var = 1.00 | |
| 0.24.135.967 I 00151: f32: __fattn__-3 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.94, avg = -0.00, max = 15.11, var = 0.58 | |
| 0.24.166.255 I 00152: f32: kqv_out-3 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.94, avg = -0.00, max = 15.11, var = 0.58 | |
| 0.24.222.739 I 00153: f32: node_152 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -155.93, avg = 0.04, max = 260.70, var = 24.92 | |
| 0.24.243.147 I 00154: f32: norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.09, avg = 0.01, max = 48.68, var = 1.00 | |
| 0.24.263.204 I 00155: f32: attn_post_norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.68, avg = 0.00, max = 17.87, var = 0.08 | |
| 0.24.282.978 I 00156: f32: attn_out-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -55.18, avg = 0.01, max = 103.21, var = 2.45 | |
| 0.24.302.945 I 00157: f32: norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.88, avg = 0.01, max = 58.90, var = 1.00 | |
| 0.24.322.980 I 00158: f32: ffn_norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -124.24, avg = 0.01, max = 28.34, var = 0.36 | |
| 0.24.498.660 I 00159: f32: ffn_gate-3 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.39, avg = -2.30, max = 12.47, var = 1.40 | |
| 0.24.674.237 I 00160: f32: ffn_up-3 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.19, avg = -0.01, max = 49.13, var = 1.07 | |
| 0.24.755.369 I 00161: f32: ffn_geglu-3 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -104.05, avg = -0.00, max = 280.50, var = 0.19 | |
| 0.24.872.400 I 00162: f32: ffn_out-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -504.85, avg = -0.01, max = 331.41, var = 7.05 | |
| 0.24.892.603 I 00163: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.25, avg = -0.01, max = 16.40, var = 1.00 | |
| 0.24.912.253 I 00164: f32: ffn_post_norm-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -4.07, avg = 0.00, max = 5.45, var = 0.01 | |
| 0.24.932.070 I 00165: f32: node_164 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.40, avg = 0.01, max = 103.66, var = 2.63 | |
| 0.24.952.659 I 00166: f32: l_out-3 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -55.83, avg = 0.01, max = 100.82, var = 2.49 | |
| 0.24.972.698 I 00167: f32: norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.92, avg = 0.01, max = 59.30, var = 1.00 | |
| 0.24.992.661 I 00168: f32: attn_norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.94, avg = 0.00, max = 99.61, var = 0.19 | |
| 0.25.068.637 I 00169: f32: Qcur-4 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.37, avg = -0.01, max = 21.61, var = 1.20 | |
| 0.25.098.861 I 00170: f32: Qcur-4 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.37, avg = -0.01, max = 21.61, var = 1.20 | |
| 0.25.130.017 I 00171: f32: norm-4 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.89, avg = -0.01, max = 11.11, var = 1.00 | |
| 0.25.160.548 I 00172: f32: Qcur_normed-4 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.23, avg = -0.01, max = 11.46, var = 1.06 | |
| 0.25.191.800 I 00173: f32: Qcur_pos-4 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.18, avg = -0.01, max = 11.46, var = 1.06 | |
| 0.25.228.034 I 00174: f32: Kcur-4 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.24, avg = -0.01, max = 24.10, var = 2.24 | |
| 0.25.243.159 I 00175: f32: Kcur-4 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.24, avg = -0.01, max = 24.10, var = 2.24 | |
| 0.25.258.842 I 00176: f32: norm-4 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.09, avg = -0.01, max = 5.97, var = 1.00 | |
| 0.25.274.242 I 00177: f32: Kcur_normed-4 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.86, avg = -0.00, max = 0.72, var = 0.01 | |
| 0.25.289.802 I 00178: f32: Kcur_pos-4 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.85, avg = -0.00, max = 0.73, var = 0.01 | |
| 0.25.323.507 I 00179: f32: Vcur-4 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.94, avg = -0.00, max = 104.17, var = 0.55 | |
| 0.25.338.640 I 00180: f32: Vcur-4 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.94, avg = -0.00, max = 104.17, var = 0.55 | |
| 0.25.354.350 I 00181: f32: Vcur_normed-4 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 1.00 | |
| 0.25.369.506 I 00182: f32: Kcur_pos-4 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.85, avg = -0.00, max = 0.73, var = 0.01 | |
| 0.25.413.504 I 00183: f16: cache_k_l4 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.85, avg = -0.00, max = 0.73, var = 0.00 | |
| 0.25.428.561 I 00184: f32: Vcur_normed-4 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 1.00 | |
| 0.25.472.404 I 00185: f16: cache_v_l4 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 0.33 | |
| 0.25.502.476 I 00186: f32: Qcur_pos-4 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.18, avg = -0.01, max = 11.46, var = 1.06 | |
| 0.25.532.695 I 00187: f32: Qcur_pos-4 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.18, avg = -0.01, max = 11.46, var = 1.06 | |
| 0.25.547.373 I 00188: f16: cache_k_l4 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.85, avg = -0.00, max = 0.73, var = 0.01 | |
| 0.25.562.004 I 00189: f16: cache_k_l4 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.85, avg = -0.00, max = 0.73, var = 0.01 | |
| 0.25.576.577 I 00190: f16: cache_v_l4 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 1.00 | |
| 0.25.591.144 I 00191: f16: cache_v_l4 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 1.00 | |
| 0.25.631.899 I 00192: f32: __fattn__-4 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 0.24 | |
| 0.25.662.161 I 00193: f32: kqv_out-4 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.93, var = 0.24 | |
| 0.25.718.895 I 00194: f32: node_193 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.34, avg = -0.02, max = 34.09, var = 3.30 | |
| 0.25.739.289 I 00195: f32: norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.61, avg = -0.01, max = 14.92, var = 1.00 | |
| 0.25.758.938 I 00196: f32: attn_post_norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.40, avg = 0.00, max = 0.62, var = 0.00 | |
| 0.25.778.721 I 00197: f32: attn_out-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -56.15, avg = 0.01, max = 100.77, var = 2.51 | |
| 0.25.798.787 I 00198: f32: norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.94, avg = 0.01, max = 59.25, var = 1.00 | |
| 0.25.818.819 I 00199: f32: ffn_norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -43.48, avg = 0.00, max = 71.11, var = 1.04 | |
| 0.25.994.591 I 00200: f32: ffn_gate-4 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.39, avg = -2.93, max = 17.63, var = 2.45 | |
| 0.26.170.343 I 00201: f32: ffn_up-4 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.45, avg = -0.02, max = 23.88, var = 2.37 | |
| 0.26.251.581 I 00202: f32: ffn_geglu-4 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -333.62, avg = 0.00, max = 245.14, var = 0.71 | |
| 0.26.381.780 I 00203: f32: ffn_out-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.67, avg = 0.03, max = 115.02, var = 7.14 | |
| 0.26.402.186 I 00204: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.42, avg = 0.01, max = 16.92, var = 1.00 | |
| 0.26.421.863 I 00205: f32: ffn_post_norm-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -3.73, avg = 0.00, max = 5.52, var = 0.01 | |
| 0.26.441.696 I 00206: f32: node_205 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.65, avg = 0.01, max = 100.91, var = 2.73 | |
| 0.26.461.803 I 00207: f32: l_out-4 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.95, avg = 0.01, max = 99.73, var = 2.66 | |
| 0.26.481.769 I 00208: f32: norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.39, avg = 0.01, max = 58.90, var = 1.00 | |
| 0.26.501.768 I 00209: f32: attn_norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.22, avg = 0.02, max = 1119.08, var = 3.13 | |
| 0.26.636.128 I 00210: f32: Qcur-5 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -159.75, avg = -0.01, max = 160.26, var = 16.41 | |
| 0.26.696.546 I 00211: f32: Qcur-5 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -159.75, avg = -0.01, max = 160.26, var = 16.41 | |
| 0.26.759.205 I 00212: f32: norm-5 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.44, avg = -0.00, max = 12.11, var = 1.00 | |
| 0.26.820.321 I 00213: f32: Qcur_normed-5 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.67, avg = -0.00, max = 12.30, var = 1.03 | |
| 0.26.881.513 I 00214: f32: Qcur_pos-5 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.67, avg = 0.00, max = 12.30, var = 1.03 | |
| 0.26.898.589 I 00215: f32: Vcur-5 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -346.36, avg = 0.02, max = 346.06, var = 11.72 | |
| 0.26.906.158 I 00216: f32: Vcur-5 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -346.36, avg = 0.02, max = 346.06, var = 11.72 | |
| 0.26.914.236 I 00217: f32: norm-5 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 1.00 | |
| 0.26.921.887 I 00218: f32: Kcur_normed-5 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.98, avg = 0.00, max = 0.98, var = 0.00 | |
| 0.26.930.050 I 00219: f32: Kcur_pos-5 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.98, avg = 0.00, max = 0.98, var = 0.00 | |
| 0.26.937.603 I 00220: f32: Vcur-5 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -346.36, avg = 0.02, max = 346.06, var = 11.72 | |
| 0.26.945.685 I 00221: f32: Vcur_normed-5 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 1.00 | |
| 0.26.953.270 I 00222: f32: Kcur_pos-5 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.98, avg = 0.00, max = 0.98, var = 0.00 | |
| 0.27.011.521 I 00223: f16: cache_k_l5 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.98, avg = 0.00, max = 0.98, var = 0.00 | |
| 0.27.018.823 I 00224: f32: Vcur_normed-5 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 1.00 | |
| 0.27.077.036 I 00225: f16: cache_v_l5 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.00, max = 15.94, var = 0.12 | |
| 0.27.137.185 I 00226: f32: Qcur_pos-5 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.67, avg = 0.00, max = 12.30, var = 1.03 | |
| 0.27.197.621 I 00227: f32: Qcur_pos-5 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.67, avg = 0.00, max = 12.30, var = 1.03 | |
| 0.27.205.018 I 00228: f16: cache_k_l5 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.98, avg = 0.00, max = 0.98, var = 0.00 | |
| 0.27.212.355 I 00229: f16: cache_k_l5 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.98, avg = 0.00, max = 0.98, var = 0.00 | |
| 0.27.219.648 I 00230: f16: cache_v_l5 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 1.00 | |
| 0.27.226.944 I 00231: f16: cache_v_l5 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 1.00 | |
| 0.27.306.035 I 00232: f32: __fattn__-5 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 0.56 | |
| 0.27.366.517 I 00233: f32: kqv_out-5 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.94, var = 0.56 | |
| 0.27.459.492 I 00234: f32: node_233 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -241.13, avg = 0.03, max = 400.06, var = 29.42 | |
| 0.27.479.888 I 00235: f32: norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.24, avg = 0.01, max = 58.32, var = 1.00 | |
| 0.27.499.953 I 00236: f32: attn_post_norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.11, avg = 0.01, max = 61.50, var = 0.91 | |
| 0.27.519.804 I 00237: f32: attn_out-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -93.96, avg = 0.03, max = 158.15, var = 6.09 | |
| 0.27.539.744 I 00238: f32: norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.18, avg = 0.01, max = 61.57, var = 1.00 | |
| 0.27.559.679 I 00239: f32: ffn_norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.05, avg = 0.01, max = 142.92, var = 0.40 | |
| 0.27.735.341 I 00240: f32: ffn_gate-5 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.70, avg = -2.33, max = 30.67, var = 1.59 | |
| 0.27.910.964 I 00241: f32: ffn_up-5 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.12, avg = -0.01, max = 36.27, var = 1.51 | |
| 0.27.992.260 I 00242: f32: ffn_geglu-5 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1751.77, avg = 0.00, max = 110.24, var = 0.45 | |
| 0.28.109.106 I 00243: f32: ffn_out-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -280.62, avg = -0.01, max = 341.64, var = 3.08 | |
| 0.28.129.504 I 00244: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = -0.01, max = 24.49, var = 1.00 | |
| 0.28.149.153 I 00245: f32: ffn_post_norm-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -3.68, avg = 0.00, max = 6.51, var = 0.02 | |
| 0.28.168.979 I 00246: f32: node_245 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.74, avg = 0.03, max = 163.86, var = 6.46 | |
| 0.28.189.000 I 00247: f32: l_out-5 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -85.03, avg = 0.03, max = 144.02, var = 4.99 | |
| 0.28.208.972 I 00248: f32: norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.11, avg = 0.01, max = 61.59, var = 1.00 | |
| 0.28.228.913 I 00249: f32: attn_norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.95, avg = 0.01, max = 1433.97, var = 2.15 | |
| 0.28.296.141 I 00250: f32: Qcur-6 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -126.18, avg = -0.05, max = 139.71, var = 12.31 | |
| 0.28.326.362 I 00251: f32: Qcur-6 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -126.18, avg = -0.05, max = 139.71, var = 12.31 | |
| 0.28.357.471 I 00252: f32: norm-6 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.00, avg = -0.01, max = 13.16, var = 1.00 | |
| 0.28.387.985 I 00253: f32: Qcur_normed-6 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.89, avg = -0.01, max = 13.05, var = 0.98 | |
| 0.28.418.741 I 00254: f32: Qcur_pos-6 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.02, avg = -0.01, max = 13.06, var = 0.98 | |
| 0.28.452.820 I 00255: f32: Kcur-6 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -336.94, avg = -0.04, max = 416.61, var = 17.34 | |
| 0.28.467.947 I 00256: f32: Kcur-6 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -336.94, avg = -0.04, max = 416.61, var = 17.34 | |
| 0.28.483.755 I 00257: f32: norm-6 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.27, avg = -0.01, max = 13.75, var = 1.00 | |
| 0.28.499.163 I 00258: f32: Kcur_normed-6 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = -0.00, max = 1.73, var = 0.02 | |
| 0.28.514.727 I 00259: f32: Kcur_pos-6 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = -0.00, max = 1.73, var = 0.02 | |
| 0.28.548.456 I 00260: f32: Vcur-6 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -853.00, avg = 0.01, max = 855.08, var = 11.11 | |
| 0.28.563.585 I 00261: f32: Vcur-6 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -853.00, avg = 0.01, max = 855.08, var = 11.11 | |
| 0.28.579.268 I 00262: f32: Vcur_normed-6 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 1.00 | |
| 0.28.594.424 I 00263: f32: Kcur_pos-6 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = -0.00, max = 1.73, var = 0.02 | |
| 0.28.638.435 I 00264: f16: cache_k_l6 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = -0.00, max = 1.73, var = 0.01 | |
| 0.28.653.408 I 00265: f32: Vcur_normed-6 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 1.00 | |
| 0.28.697.261 I 00266: f16: cache_v_l6 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 0.33 | |
| 0.28.727.235 I 00267: f32: Qcur_pos-6 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.02, avg = -0.01, max = 13.06, var = 0.98 | |
| 0.28.757.457 I 00268: f32: Qcur_pos-6 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.02, avg = -0.01, max = 13.06, var = 0.98 | |
| 0.28.772.122 I 00269: f16: cache_k_l6 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = -0.00, max = 1.73, var = 0.02 | |
| 0.28.786.736 I 00270: f16: cache_k_l6 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = -0.00, max = 1.73, var = 0.02 | |
| 0.28.801.303 I 00271: f16: cache_v_l6 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 1.00 | |
| 0.28.815.864 I 00272: f16: cache_v_l6 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 1.00 | |
| 0.28.856.463 I 00273: f32: __fattn__-6 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.01, max = 15.97, var = 0.50 | |
| 0.28.886.707 I 00274: f32: kqv_out-6 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.01, max = 15.97, var = 0.50 | |
| 0.28.943.489 I 00275: f32: node_274 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -165.56, avg = 0.03, max = 265.66, var = 12.24 | |
| 0.28.963.913 I 00276: f32: norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.00, avg = 0.01, max = 51.16, var = 1.00 | |
| 0.28.984.096 I 00277: f32: attn_post_norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.22, avg = 0.01, max = 39.97, var = 0.42 | |
| 0.29.003.982 I 00278: f32: attn_out-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -106.99, avg = 0.03, max = 182.50, var = 7.85 | |
| 0.29.024.068 I 00279: f32: norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.30, avg = 0.01, max = 61.71, var = 1.00 | |
| 0.29.044.095 I 00280: f32: ffn_norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.20, avg = -0.00, max = 21.57, var = 1.28 | |
| 0.29.219.666 I 00281: f32: ffn_gate-6 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.35, avg = -4.32, max = 15.87, var = 3.44 | |
| 0.29.395.259 I 00282: f32: ffn_up-6 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.83, avg = 0.00, max = 17.79, var = 3.93 | |
| 0.29.475.868 I 00283: f32: ffn_geglu-6 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -361.01, avg = 0.00, max = 213.84, var = 0.37 | |
| 0.29.592.756 I 00284: f32: ffn_out-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.99, avg = 0.04, max = 132.60, var = 4.20 | |
| 0.29.613.148 I 00285: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.44, avg = 0.03, max = 20.72, var = 1.00 | |
| 0.29.632.924 I 00286: f32: ffn_post_norm-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -5.44, avg = 0.00, max = 7.81, var = 0.03 | |
| 0.29.652.725 I 00287: f32: node_286 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -111.06, avg = 0.03, max = 188.24, var = 8.35 | |
| 0.29.672.787 I 00288: f32: l_out-6 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -101.95, avg = 0.03, max = 172.80, var = 7.04 | |
| 0.29.692.779 I 00289: f32: norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.52, avg = 0.01, max = 61.80, var = 1.00 | |
| 0.29.712.679 I 00290: f32: attn_norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.40, avg = 0.01, max = 4448.75, var = 11.19 | |
| 0.29.779.945 I 00291: f32: Qcur-7 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -420.98, avg = -0.03, max = 466.89, var = 41.54 | |
| 0.29.810.192 I 00292: f32: Qcur-7 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -420.98, avg = -0.03, max = 466.89, var = 41.54 | |
| 0.29.841.283 I 00293: f32: norm-7 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.72, avg = -0.01, max = 12.64, var = 1.00 | |
| 0.29.871.801 I 00294: f32: Qcur_normed-7 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.18, avg = -0.01, max = 13.13, var = 1.08 | |
| 0.29.902.576 I 00295: f32: Qcur_pos-7 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.57, avg = -0.01, max = 13.42, var = 1.08 | |
| 0.29.936.596 I 00296: f32: Kcur-7 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -916.46, avg = -0.00, max = 615.60, var = 50.98 | |
| 0.29.951.717 I 00297: f32: Kcur-7 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -916.46, avg = -0.00, max = 615.60, var = 50.98 | |
| 0.29.967.398 I 00298: f32: norm-7 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.04, avg = -0.00, max = 10.36, var = 1.00 | |
| 0.29.982.796 I 00299: f32: Kcur_normed-7 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.81, avg = -0.00, max = 1.24, var = 0.01 | |
| 0.29.998.383 I 00300: f32: Kcur_pos-7 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.81, avg = -0.00, max = 1.24, var = 0.01 | |
| 0.30.032.164 I 00301: f32: Vcur-7 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1501.60, avg = 0.07, max = 2164.02, var = 40.85 | |
| 0.30.047.295 I 00302: f32: Vcur-7 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1501.60, avg = 0.07, max = 2164.02, var = 40.85 | |
| 0.30.062.958 I 00303: f32: Vcur_normed-7 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.02, max = 16.00, var = 1.00 | |
| 0.30.078.121 I 00304: f32: Kcur_pos-7 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.81, avg = -0.00, max = 1.24, var = 0.01 | |
| 0.30.122.143 I 00305: f16: cache_k_l7 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.81, avg = -0.00, max = 1.25, var = 0.00 | |
| 0.30.137.012 I 00306: f32: Vcur_normed-7 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.02, max = 16.00, var = 1.00 | |
| 0.30.180.888 I 00307: f16: cache_v_l7 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 16.00, var = 0.33 | |
| 0.30.210.946 I 00308: f32: Qcur_pos-7 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.57, avg = -0.01, max = 13.42, var = 1.08 | |
| 0.30.241.165 I 00309: f32: Qcur_pos-7 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.57, avg = -0.01, max = 13.42, var = 1.08 | |
| 0.30.255.801 I 00310: f16: cache_k_l7 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.81, avg = -0.00, max = 1.25, var = 0.01 | |
| 0.30.270.445 I 00311: f16: cache_k_l7 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.81, avg = -0.00, max = 1.25, var = 0.01 | |
| 0.30.285.023 I 00312: f16: cache_v_l7 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.02, max = 16.00, var = 1.00 | |
| 0.30.299.582 I 00313: f16: cache_v_l7 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.02, max = 16.00, var = 1.00 | |
| 0.30.340.251 I 00314: f32: __fattn__-7 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.01, max = 16.00, var = 0.72 | |
| 0.30.370.486 I 00315: f32: kqv_out-7 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.01, max = 16.00, var = 0.72 | |
| 0.30.427.435 I 00316: f32: node_315 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -123.85, avg = 0.04, max = 224.54, var = 12.24 | |
| 0.30.447.816 I 00317: f32: norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.21, avg = 0.01, max = 53.01, var = 1.00 | |
| 0.30.467.995 I 00318: f32: attn_post_norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.43, avg = 0.01, max = 55.08, var = 0.77 | |
| 0.30.487.871 I 00319: f32: attn_out-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.61, avg = 0.04, max = 227.55, var = 11.70 | |
| 0.30.507.987 I 00320: f32: norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.57, avg = 0.01, max = 62.80, var = 1.00 | |
| 0.30.528.030 I 00321: f32: ffn_norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.56, avg = 0.00, max = 17.42, var = 0.32 | |
| 0.30.703.681 I 00322: f32: ffn_gate-7 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.98, avg = -1.85, max = 10.49, var = 1.46 | |
| 0.30.879.426 I 00323: f32: ffn_up-7 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.97, avg = 0.00, max = 21.43, var = 1.34 | |
| 0.30.960.442 I 00324: f32: ffn_geglu-7 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.11, avg = -0.00, max = 224.86, var = 0.09 | |
| 0.31.077.191 I 00325: f32: ffn_out-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.79, avg = -0.00, max = 44.83, var = 0.91 | |
| 0.31.097.584 I 00326: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.77, avg = -0.00, max = 32.59, var = 1.00 | |
| 0.31.117.221 I 00327: f32: ffn_post_norm-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.44, avg = 0.00, max = 19.86, var = 0.09 | |
| 0.31.137.040 I 00328: f32: node_327 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -136.51, avg = 0.04, max = 243.94, var = 12.88 | |
| 0.31.157.531 I 00329: f32: l_out-7 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -116.78, avg = 0.03, max = 208.69, var = 9.42 | |
| 0.31.177.608 I 00330: f32: norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.24, avg = 0.01, max = 62.91, var = 1.00 | |
| 0.31.197.587 I 00331: f32: attn_norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.09, avg = 0.01, max = 1175.10, var = 1.57 | |
| 0.31.264.950 I 00332: f32: Qcur-8 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -165.13, avg = 0.01, max = 226.97, var = 8.27 | |
| 0.31.295.177 I 00333: f32: Qcur-8 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -165.13, avg = 0.01, max = 226.97, var = 8.27 | |
| 0.31.326.339 I 00334: f32: norm-8 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.35, avg = 0.01, max = 11.98, var = 1.00 | |
| 0.31.356.645 I 00335: f32: Qcur_normed-8 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.16, avg = 0.01, max = 11.79, var = 0.97 | |
| 0.31.386.746 I 00336: f32: Qcur_pos-8 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.33, avg = -0.00, max = 12.28, var = 0.97 | |
| 0.31.421.052 I 00337: f32: Kcur-8 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -273.76, avg = -0.01, max = 255.63, var = 13.11 | |
| 0.31.436.386 I 00338: f32: Kcur-8 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -273.76, avg = -0.01, max = 255.63, var = 13.11 | |
| 0.31.451.971 I 00339: f32: norm-8 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.86, avg = 0.00, max = 10.03, var = 1.00 | |
| 0.31.467.241 I 00340: f32: Kcur_normed-8 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.27, var = 0.02 | |
| 0.31.482.689 I 00341: f32: Kcur_pos-8 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.27, var = 0.02 | |
| 0.31.527.411 I 00342: f32: Vcur-8 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -664.52, avg = 0.01, max = 605.83, var = 5.82 | |
| 0.31.543.005 I 00343: f32: Vcur-8 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -664.52, avg = 0.01, max = 605.83, var = 5.82 | |
| 0.31.558.382 I 00344: f32: Vcur_normed-8 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.31.573.066 I 00345: f32: Kcur_pos-8 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.27, var = 0.02 | |
| 0.31.617.000 I 00346: f16: cache_k_l8 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.50, avg = 0.00, max = 1.27, var = 0.01 | |
| 0.31.631.665 I 00347: f32: Vcur_normed-8 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.31.675.484 I 00348: f16: cache_v_l8 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 0.33 | |
| 0.31.704.747 I 00349: f32: Qcur_pos-8 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.33, avg = -0.00, max = 12.28, var = 0.97 | |
| 0.31.733.973 I 00350: f32: Qcur_pos-8 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.33, avg = -0.00, max = 12.28, var = 0.97 | |
| 0.31.748.586 I 00351: f16: cache_k_l8 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.50, avg = 0.00, max = 1.27, var = 0.02 | |
| 0.31.763.191 I 00352: f16: cache_k_l8 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.50, avg = 0.00, max = 1.27, var = 0.02 | |
| 0.31.777.743 I 00353: f16: cache_v_l8 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.31.792.289 I 00354: f16: cache_v_l8 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.31.834.078 I 00355: f32: __fattn__-8 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 0.54 | |
| 0.31.864.171 I 00356: f32: kqv_out-8 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 0.54 | |
| 0.31.929.704 I 00357: f32: node_356 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -66.63, avg = 0.01, max = 120.78, var = 5.75 | |
| 0.31.949.682 I 00358: f32: norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.86, avg = 0.00, max = 48.55, var = 1.00 | |
| 0.31.969.186 I 00359: f32: attn_post_norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.91, avg = 0.00, max = 39.26, var = 0.24 | |
| 0.31.988.982 I 00360: f32: attn_out-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -135.90, avg = 0.03, max = 241.53, var = 12.34 | |
| 0.32.008.940 I 00361: f32: norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.54, avg = 0.01, max = 63.01, var = 1.00 | |
| 0.32.029.952 I 00362: f32: ffn_norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.21, avg = 0.00, max = 14.84, var = 0.40 | |
| 0.32.206.410 I 00363: f32: ffn_gate-8 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.47, avg = -1.75, max = 10.88, var = 1.32 | |
| 0.32.386.276 I 00364: f32: ffn_up-8 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.51, avg = -0.00, max = 16.11, var = 1.34 | |
| 0.32.468.856 I 00365: f32: ffn_geglu-8 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -97.55, avg = -0.00, max = 81.53, var = 0.14 | |
| 0.32.585.740 I 00366: f32: ffn_out-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.25, avg = -0.01, max = 47.26, var = 2.20 | |
| 0.32.606.517 I 00367: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.85, avg = -0.00, max = 36.43, var = 1.00 | |
| 0.32.626.343 I 00368: f32: ffn_post_norm-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.10, avg = 0.00, max = 29.17, var = 0.14 | |
| 0.32.648.135 I 00369: f32: node_368 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -148.78, avg = 0.04, max = 267.71, var = 14.26 | |
| 0.32.669.597 I 00370: f32: l_out-8 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -129.60, avg = 0.03, max = 233.20, var = 10.82 | |
| 0.32.689.601 I 00371: f32: norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.12, avg = 0.01, max = 63.36, var = 1.00 | |
| 0.32.709.755 I 00372: f32: attn_norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -62.73, avg = 0.02, max = 2159.07, var = 4.38 | |
| 0.32.799.235 I 00373: f32: Qcur-9 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -502.94, avg = 0.03, max = 243.52, var = 28.08 | |
| 0.32.828.738 I 00374: f32: Qcur-9 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -502.94, avg = 0.03, max = 243.52, var = 28.08 | |
| 0.32.859.249 I 00375: f32: norm-9 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.52, avg = 0.01, max = 12.08, var = 1.00 | |
| 0.32.888.872 I 00376: f32: Qcur_normed-9 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.62, avg = 0.01, max = 12.18, var = 1.02 | |
| 0.32.920.309 I 00377: f32: Qcur_pos-9 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.43, avg = 0.00, max = 12.86, var = 1.02 | |
| 0.32.962.383 I 00378: f32: Kcur-9 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -345.95, avg = -0.03, max = 464.81, var = 50.61 | |
| 0.32.977.491 I 00379: f32: Kcur-9 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -345.95, avg = -0.03, max = 464.81, var = 50.61 | |
| 0.32.992.809 I 00380: f32: norm-9 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.61, avg = 0.00, max = 12.49, var = 1.00 | |
| 0.33.009.039 I 00381: f32: Kcur_normed-9 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.20, avg = 0.00, max = 1.56, var = 0.02 | |
| 0.33.024.441 I 00382: f32: Kcur_pos-9 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.20, avg = 0.00, max = 1.56, var = 0.02 | |
| 0.33.058.532 I 00383: f32: Vcur-9 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1183.89, avg = 0.00, max = 856.17, var = 16.40 | |
| 0.33.073.805 I 00384: f32: Vcur-9 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1183.89, avg = 0.00, max = 856.17, var = 16.40 | |
| 0.33.089.058 I 00385: f32: Vcur_normed-9 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.33.103.752 I 00386: f32: Kcur_pos-9 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.20, avg = 0.00, max = 1.56, var = 0.02 | |
| 0.33.147.681 I 00387: f16: cache_k_l9 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.20, avg = 0.00, max = 1.55, var = 0.01 | |
| 0.33.162.358 I 00388: f32: Vcur_normed-9 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.33.206.198 I 00389: f16: cache_v_l9 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 0.33 | |
| 0.33.235.520 I 00390: f32: Qcur_pos-9 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.43, avg = 0.00, max = 12.86, var = 1.02 | |
| 0.33.264.721 I 00391: f32: Qcur_pos-9 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.43, avg = 0.00, max = 12.86, var = 1.02 | |
| 0.33.279.372 I 00392: f16: cache_k_l9 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.20, avg = 0.00, max = 1.55, var = 0.02 | |
| 0.33.293.977 I 00393: f16: cache_k_l9 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.20, avg = 0.00, max = 1.55, var = 0.02 | |
| 0.33.308.523 I 00394: f16: cache_v_l9 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.33.323.086 I 00395: f16: cache_v_l9 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.33.365.743 I 00396: f32: __fattn__-9 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.98, var = 0.74 | |
| 0.33.395.960 I 00397: f32: kqv_out-9 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.98, var = 0.74 | |
| 0.33.452.792 I 00398: f32: node_397 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -127.38, avg = 0.00, max = 247.07, var = 13.59 | |
| 0.33.473.314 I 00399: f32: norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.78, avg = 0.00, max = 59.32, var = 1.00 | |
| 0.33.494.262 I 00400: f32: attn_post_norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.80, avg = 0.00, max = 63.49, var = 0.62 | |
| 0.33.515.471 I 00401: f32: attn_out-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -160.33, avg = 0.03, max = 289.69, var = 16.14 | |
| 0.33.535.375 I 00402: f32: norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.31, avg = 0.01, max = 64.31, var = 1.00 | |
| 0.33.555.595 I 00403: f32: ffn_norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.12, avg = -0.01, max = 44.71, var = 0.35 | |
| 0.33.745.730 I 00404: f32: ffn_gate-9 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.32, avg = -1.69, max = 19.62, var = 1.14 | |
| 0.33.937.297 I 00405: f32: ffn_up-9 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.72, avg = -0.00, max = 28.61, var = 0.90 | |
| 0.34.019.999 I 00406: f32: ffn_geglu-9 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.88, avg = 0.00, max = 561.35, var = 0.12 | |
| 0.34.152.483 I 00407: f32: ffn_out-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -114.88, avg = 0.04, max = 146.58, var = 2.30 | |
| 0.34.172.750 I 00408: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.71, avg = 0.03, max = 22.38, var = 1.00 | |
| 0.34.192.399 I 00409: f32: ffn_post_norm-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.96, avg = 0.00, max = 11.36, var = 0.04 | |
| 0.34.212.161 I 00410: f32: node_409 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -164.12, avg = 0.03, max = 299.27, var = 17.17 | |
| 0.34.232.369 I 00411: f32: l_out-9 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -141.04, avg = 0.03, max = 257.19, var = 12.68 | |
| 0.34.252.345 I 00412: f32: norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.29, avg = 0.01, max = 64.32, var = 1.00 | |
| 0.34.273.715 I 00413: f32: attn_norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.10, avg = 0.01, max = 970.11, var = 1.61 | |
| 0.34.346.955 I 00414: f32: Qcur-10 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -200.23, avg = 0.03, max = 193.37, var = 9.11 | |
| 0.34.377.152 I 00415: f32: Qcur-10 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -200.23, avg = 0.03, max = 193.37, var = 9.11 | |
| 0.34.408.761 I 00416: f32: norm-10 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.64, avg = 0.01, max = 12.13, var = 1.00 | |
| 0.34.438.457 I 00417: f32: Qcur_normed-10 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.82, avg = 0.01, max = 12.32, var = 1.03 | |
| 0.34.469.349 I 00418: f32: Qcur_pos-10 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.40, avg = 0.01, max = 12.30, var = 1.03 | |
| 0.34.504.173 I 00419: f32: Kcur-10 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -193.74, avg = -0.02, max = 261.16, var = 12.43 | |
| 0.34.519.514 I 00420: f32: Kcur-10 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -193.74, avg = -0.02, max = 261.16, var = 12.43 | |
| 0.34.535.096 I 00421: f32: norm-10 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.36, avg = -0.01, max = 13.43, var = 1.00 | |
| 0.34.550.618 I 00422: f32: Kcur_normed-10 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = -0.00, max = 1.66, var = 0.02 | |
| 0.34.566.303 I 00423: f32: Kcur_pos-10 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = -0.00, max = 1.66, var = 0.02 | |
| 0.34.600.756 I 00424: f32: Vcur-10 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -735.87, avg = 0.01, max = 805.36, var = 7.15 | |
| 0.34.615.863 I 00425: f32: Vcur-10 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -735.87, avg = 0.01, max = 805.36, var = 7.15 | |
| 0.34.631.256 I 00426: f32: Vcur_normed-10 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.34.646.391 I 00427: f32: Kcur_pos-10 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = -0.00, max = 1.66, var = 0.02 | |
| 0.34.690.295 I 00428: f16: cache_k_l10 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = -0.00, max = 1.66, var = 0.01 | |
| 0.34.705.173 I 00429: f32: Vcur_normed-10 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.34.749.004 I 00430: f16: cache_v_l10 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 16.00, var = 0.33 | |
| 0.34.778.956 I 00431: f32: Qcur_pos-10 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.40, avg = 0.01, max = 12.30, var = 1.03 | |
| 0.34.809.158 I 00432: f32: Qcur_pos-10 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.40, avg = 0.01, max = 12.30, var = 1.03 | |
| 0.34.823.814 I 00433: f16: cache_k_l10 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = -0.00, max = 1.66, var = 0.02 | |
| 0.34.838.417 I 00434: f16: cache_k_l10 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.28, avg = -0.00, max = 1.66, var = 0.02 | |
| 0.34.852.970 I 00435: f16: cache_v_l10 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.34.867.570 I 00436: f16: cache_v_l10 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.34.911.119 I 00437: f32: __fattn__-10 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 16.00, var = 0.59 | |
| 0.34.941.353 I 00438: f32: kqv_out-10 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 16.00, var = 0.59 | |
| 0.34.998.381 I 00439: f32: node_438 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -158.69, avg = 0.01, max = 259.57, var = 10.21 | |
| 0.35.019.148 I 00440: f32: norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.68, avg = 0.00, max = 53.15, var = 1.00 | |
| 0.35.038.685 I 00441: f32: attn_post_norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.65, avg = 0.00, max = 37.99, var = 0.30 | |
| 0.35.060.083 I 00442: f32: attn_out-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -163.69, avg = 0.03, max = 291.92, var = 16.44 | |
| 0.35.080.682 I 00443: f32: norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.59, avg = 0.01, max = 64.10, var = 1.00 | |
| 0.35.100.042 I 00444: f32: ffn_norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -40.34, avg = -0.00, max = 69.32, var = 0.89 | |
| 0.35.283.618 I 00445: f32: ffn_gate-10 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.89, avg = -2.88, max = 22.83, var = 1.60 | |
| 0.35.462.042 I 00446: f32: ffn_up-10 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -50.94, avg = 0.01, max = 20.00, var = 1.35 | |
| 0.35.544.688 I 00447: f32: ffn_geglu-10 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1163.12, avg = 0.00, max = 62.39, var = 0.22 | |
| 0.35.662.991 I 00448: f32: ffn_out-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -270.36, avg = -0.01, max = 293.60, var = 2.67 | |
| 0.35.683.357 I 00449: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.27, avg = -0.01, max = 18.75, var = 1.00 | |
| 0.35.704.164 I 00450: f32: ffn_post_norm-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.55, avg = 0.00, max = 9.81, var = 0.04 | |
| 0.35.725.088 I 00451: f32: node_450 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -168.09, avg = 0.03, max = 299.73, var = 17.35 | |
| 0.35.745.290 I 00452: f32: l_out-10 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -153.65, avg = 0.03, max = 273.98, var = 14.50 | |
| 0.35.765.078 I 00453: f32: norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.79, avg = 0.01, max = 64.11, var = 1.00 | |
| 0.35.785.019 I 00454: f32: attn_norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.96, avg = 0.02, max = 1316.71, var = 5.16 | |
| 0.35.919.824 I 00455: f32: Qcur-11 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -209.65, avg = -0.03, max = 150.49, var = 17.52 | |
| 0.35.980.204 I 00456: f32: Qcur-11 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -209.65, avg = -0.03, max = 150.49, var = 17.52 | |
| 0.36.042.495 I 00457: f32: norm-11 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.32, avg = -0.01, max = 14.03, var = 1.00 | |
| 0.36.104.139 I 00458: f32: Qcur_normed-11 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.80, avg = -0.01, max = 14.58, var = 1.08 | |
| 0.36.166.376 I 00459: f32: Qcur_pos-11 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.03, avg = -0.00, max = 17.33, var = 1.08 | |
| 0.36.184.213 I 00460: f32: Vcur-11 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -644.64, avg = -0.02, max = 469.26, var = 13.01 | |
| 0.36.191.817 I 00461: f32: Vcur-11 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -644.64, avg = -0.02, max = 469.26, var = 13.01 | |
| 0.36.199.991 I 00462: f32: norm-11 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.27, avg = -0.01, max = 15.60, var = 1.00 | |
| 0.36.207.490 I 00463: f32: Kcur_normed-11 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 0.94, var = 0.00 | |
| 0.36.215.104 I 00464: f32: Kcur_pos-11 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 0.94, var = 0.00 | |
| 0.36.223.000 I 00465: f32: Vcur-11 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -644.64, avg = -0.02, max = 469.26, var = 13.01 | |
| 0.36.230.764 I 00466: f32: Vcur_normed-11 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.27, avg = -0.01, max = 15.60, var = 1.00 | |
| 0.36.238.827 I 00467: f32: Kcur_pos-11 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 0.94, var = 0.00 | |
| 0.36.297.125 I 00468: f16: cache_k_l11 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 0.94, var = 0.00 | |
| 0.36.304.470 I 00469: f32: Vcur_normed-11 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.27, avg = -0.01, max = 15.60, var = 1.00 | |
| 0.36.362.691 I 00470: f16: cache_v_l11 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.27, avg = -0.00, max = 15.59, var = 0.12 | |
| 0.36.421.721 I 00471: f32: Qcur_pos-11 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.03, avg = -0.00, max = 17.33, var = 1.08 | |
| 0.36.482.102 I 00472: f32: Qcur_pos-11 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.03, avg = -0.00, max = 17.33, var = 1.08 | |
| 0.36.489.450 I 00473: f16: cache_k_l11 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 0.94, var = 0.00 | |
| 0.36.496.777 I 00474: f16: cache_k_l11 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 0.94, var = 0.00 | |
| 0.36.504.067 I 00475: f16: cache_v_l11 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.27, avg = -0.01, max = 15.59, var = 1.00 | |
| 0.36.511.371 I 00476: f16: cache_v_l11 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.27, avg = -0.01, max = 15.59, var = 1.00 | |
| 0.36.592.257 I 00477: f32: __fattn__-11 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.22, avg = -0.02, max = 15.59, var = 0.67 | |
| 0.36.652.702 I 00478: f32: kqv_out-11 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.22, avg = -0.02, max = 15.59, var = 0.67 | |
| 0.36.753.753 I 00479: f32: node_478 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -163.64, avg = -0.00, max = 319.10, var = 19.77 | |
| 0.36.774.167 I 00480: f32: norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.31, avg = -0.00, max = 61.12, var = 1.00 | |
| 0.36.794.279 I 00481: f32: attn_post_norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -40.25, avg = 0.01, max = 86.90, var = 1.33 | |
| 0.36.814.081 I 00482: f32: attn_out-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -192.73, avg = 0.04, max = 360.17, var = 24.06 | |
| 0.36.835.780 I 00483: f32: norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.17, avg = 0.01, max = 65.23, var = 1.00 | |
| 0.36.856.424 I 00484: f32: ffn_norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -227.40, avg = -0.02, max = 48.74, var = 1.97 | |
| 0.37.043.379 I 00485: f32: ffn_gate-11 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.23, avg = -3.38, max = 30.87, var = 2.25 | |
| 0.37.221.059 I 00486: f32: ffn_up-11 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.08, avg = -0.01, max = 56.99, var = 2.36 | |
| 0.37.302.050 I 00487: f32: ffn_geglu-11 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -169.70, avg = 0.00, max = 1759.44, var = 0.52 | |
| 0.37.425.728 I 00488: f32: ffn_out-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -354.11, avg = -0.02, max = 335.36, var = 4.35 | |
| 0.37.446.643 I 00489: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.57, avg = -0.01, max = 16.86, var = 1.00 | |
| 0.37.466.303 I 00490: f32: ffn_post_norm-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.00, avg = -0.00, max = 9.88, var = 0.05 | |
| 0.37.487.793 I 00491: f32: node_490 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -195.63, avg = 0.04, max = 365.00, var = 25.14 | |
| 0.37.508.668 I 00492: f32: l_out-11 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -155.13, avg = 0.03, max = 289.44, var = 15.81 | |
| 0.37.528.625 I 00493: f32: norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.28, avg = 0.01, max = 65.23, var = 1.00 | |
| 0.37.549.983 I 00494: f32: attn_norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -40.69, avg = 0.00, max = 106.88, var = 0.33 | |
| 0.37.625.280 I 00495: f32: Qcur-12 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.19, avg = -0.01, max = 36.41, var = 2.87 | |
| 0.37.655.489 I 00496: f32: Qcur-12 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.19, avg = -0.01, max = 36.41, var = 2.87 | |
| 0.37.687.052 I 00497: f32: norm-12 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.31, avg = -0.00, max = 14.11, var = 1.00 | |
| 0.37.717.015 I 00498: f32: Qcur_normed-12 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.09, avg = -0.00, max = 13.89, var = 0.97 | |
| 0.37.748.141 I 00499: f32: Qcur_pos-12 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.96, avg = 0.00, max = 13.94, var = 0.97 | |
| 0.37.783.182 I 00500: f32: Kcur-12 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.34, avg = -0.04, max = 25.02, var = 4.18 | |
| 0.37.798.294 I 00501: f32: Kcur-12 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.34, avg = -0.04, max = 25.02, var = 4.18 | |
| 0.37.813.989 I 00502: f32: norm-12 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.12, avg = -0.02, max = 7.55, var = 1.00 | |
| 0.37.829.738 I 00503: f32: Kcur_normed-12 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.78, avg = -0.00, max = 0.96, var = 0.02 | |
| 0.37.844.939 I 00504: f32: Kcur_pos-12 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = -0.00, max = 0.96, var = 0.02 | |
| 0.37.879.646 I 00505: f32: Vcur-12 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -223.51, avg = -0.00, max = 121.51, var = 1.08 | |
| 0.37.894.787 I 00506: f32: Vcur-12 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -223.51, avg = -0.00, max = 121.51, var = 1.08 | |
| 0.37.910.431 I 00507: f32: Vcur_normed-12 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.37.925.563 I 00508: f32: Kcur_pos-12 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = -0.00, max = 0.96, var = 0.02 | |
| 0.37.969.505 I 00509: f16: cache_k_l12 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = -0.00, max = 0.96, var = 0.01 | |
| 0.37.984.276 I 00510: f32: Vcur_normed-12 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.38.028.039 I 00511: f16: cache_v_l12 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 15.98, var = 0.33 | |
| 0.38.057.959 I 00512: f32: Qcur_pos-12 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.96, avg = 0.00, max = 13.94, var = 0.97 | |
| 0.38.088.172 I 00513: f32: Qcur_pos-12 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.96, avg = 0.00, max = 13.94, var = 0.97 | |
| 0.38.102.848 I 00514: f16: cache_k_l12 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = -0.00, max = 0.96, var = 0.02 | |
| 0.38.117.452 I 00515: f16: cache_k_l12 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = -0.00, max = 0.96, var = 0.02 | |
| 0.38.132.003 I 00516: f16: cache_v_l12 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.38.146.559 I 00517: f16: cache_v_l12 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.38.187.670 I 00518: f32: __fattn__-12 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.98, var = 0.86 | |
| 0.38.217.877 I 00519: f32: kqv_out-12 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.98, var = 0.86 | |
| 0.38.281.490 I 00520: f32: node_519 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.91, avg = -0.01, max = 278.23, var = 17.78 | |
| 0.38.301.411 I 00521: f32: norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.18, avg = -0.00, max = 63.63, var = 1.00 | |
| 0.38.320.882 I 00522: f32: attn_post_norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.82, avg = 0.01, max = 128.25, var = 3.12 | |
| 0.38.340.695 I 00523: f32: attn_out-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -207.84, avg = 0.05, max = 414.46, var = 32.12 | |
| 0.38.360.447 I 00524: f32: norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.88, avg = 0.01, max = 65.90, var = 1.00 | |
| 0.38.379.965 I 00525: f32: ffn_norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.80, avg = -0.00, max = 49.35, var = 0.36 | |
| 0.38.555.715 I 00526: f32: ffn_gate-12 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.09, avg = -1.92, max = 32.37, var = 1.18 | |
| 0.38.731.293 I 00527: f32: ffn_up-12 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.75, avg = -0.00, max = 51.03, var = 1.14 | |
| 0.38.812.295 I 00528: f32: ffn_geglu-12 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -727.68, avg = 0.00, max = 1426.43, var = 0.49 | |
| 0.38.929.279 I 00529: f32: ffn_out-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -741.72, avg = -0.00, max = 954.36, var = 8.80 | |
| 0.38.949.182 I 00530: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.60, avg = -0.00, max = 35.33, var = 1.00 | |
| 0.38.968.846 I 00531: f32: ffn_post_norm-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.21, avg = 0.00, max = 26.09, var = 0.14 | |
| 0.38.988.592 I 00532: f32: node_531 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -222.21, avg = 0.05, max = 437.73, var = 35.31 | |
| 0.39.008.724 I 00533: f32: l_out-12 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -151.90, avg = 0.03, max = 299.23, var = 16.50 | |
| 0.39.028.775 I 00534: f32: norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.91, avg = 0.01, max = 65.82, var = 1.00 | |
| 0.39.048.375 I 00535: f32: attn_norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.25, avg = 0.00, max = 45.17, var = 0.22 | |
| 0.39.115.622 I 00536: f32: Qcur-13 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.02, avg = 0.01, max = 24.51, var = 1.44 | |
| 0.39.145.275 I 00537: f32: Qcur-13 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.02, avg = 0.01, max = 24.51, var = 1.44 | |
| 0.39.175.506 I 00538: f32: norm-13 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.89, avg = 0.01, max = 11.51, var = 1.00 | |
| 0.39.205.101 I 00539: f32: Qcur_normed-13 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.30, avg = 0.01, max = 11.87, var = 1.06 | |
| 0.39.235.057 I 00540: f32: Qcur_pos-13 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.29, avg = 0.00, max = 12.44, var = 1.06 | |
| 0.39.269.326 I 00541: f32: Kcur-13 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -54.58, avg = 0.05, max = 58.52, var = 2.65 | |
| 0.39.284.336 I 00542: f32: Kcur-13 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -54.58, avg = 0.05, max = 58.52, var = 2.65 | |
| 0.39.299.596 I 00543: f32: norm-13 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.14, avg = 0.03, max = 10.60, var = 1.00 | |
| 0.39.314.581 I 00544: f32: Kcur_normed-13 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.35, avg = 0.00, max = 1.28, var = 0.01 | |
| 0.39.329.738 I 00545: f32: Kcur_pos-13 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.35, avg = 0.00, max = 1.29, var = 0.01 | |
| 0.39.363.688 I 00546: f32: Vcur-13 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -147.97, avg = -0.00, max = 155.78, var = 1.12 | |
| 0.39.378.798 I 00547: f32: Vcur-13 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -147.97, avg = -0.00, max = 155.78, var = 1.12 | |
| 0.39.394.162 I 00548: f32: Vcur_normed-13 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 1.00 | |
| 0.39.408.861 I 00549: f32: Kcur_pos-13 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.35, avg = 0.00, max = 1.29, var = 0.01 | |
| 0.39.452.889 I 00550: f16: cache_k_l13 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.35, avg = 0.00, max = 1.29, var = 0.00 | |
| 0.39.467.549 I 00551: f32: Vcur_normed-13 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 1.00 | |
| 0.39.511.441 I 00552: f16: cache_v_l13 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 0.33 | |
| 0.39.540.716 I 00553: f32: Qcur_pos-13 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.29, avg = 0.00, max = 12.44, var = 1.06 | |
| 0.39.569.999 I 00554: f32: Qcur_pos-13 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.29, avg = 0.00, max = 12.44, var = 1.06 | |
| 0.39.584.608 I 00555: f16: cache_k_l13 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.35, avg = 0.00, max = 1.29, var = 0.01 | |
| 0.39.599.210 I 00556: f16: cache_k_l13 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.35, avg = 0.00, max = 1.29, var = 0.01 | |
| 0.39.613.752 I 00557: f16: cache_v_l13 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 1.00 | |
| 0.39.628.315 I 00558: f16: cache_v_l13 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 1.00 | |
| 0.39.669.418 I 00559: f32: __fattn__-13 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.01, max = 15.95, var = 0.44 | |
| 0.39.698.725 I 00560: f32: kqv_out-13 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.01, max = 15.95, var = 0.44 | |
| 0.39.756.356 I 00561: f32: node_560 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -60.37, avg = 0.02, max = 97.02, var = 4.69 | |
| 0.39.776.714 I 00562: f32: norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.98, avg = 0.01, max = 40.89, var = 1.00 | |
| 0.39.796.270 I 00563: f32: attn_post_norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -3.55, avg = 0.00, max = 6.31, var = 0.01 | |
| 0.39.816.037 I 00564: f32: attn_out-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -155.00, avg = 0.03, max = 305.29, var = 17.09 | |
| 0.39.835.993 I 00565: f32: norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.00, avg = 0.01, max = 65.75, var = 1.00 | |
| 0.39.855.499 I 00566: f32: ffn_norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.82, avg = -0.00, max = 123.77, var = 0.68 | |
| 0.40.031.443 I 00567: f32: ffn_gate-13 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.25, avg = -2.50, max = 28.12, var = 1.71 | |
| 0.40.207.091 I 00568: f32: ffn_up-13 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -43.20, avg = -0.00, max = 49.87, var = 1.73 | |
| 0.40.288.211 I 00569: f32: ffn_geglu-13 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1108.14, avg = -0.00, max = 926.28, var = 0.49 | |
| 0.40.405.167 I 00570: f32: ffn_out-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -559.51, avg = 0.02, max = 726.50, var = 14.75 | |
| 0.40.425.090 I 00571: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.47, avg = 0.01, max = 28.06, var = 1.00 | |
| 0.40.444.749 I 00572: f32: ffn_post_norm-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.06, avg = 0.00, max = 11.35, var = 0.03 | |
| 0.40.464.327 I 00573: f32: node_572 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -160.68, avg = 0.03, max = 315.41, var = 18.15 | |
| 0.40.484.574 I 00574: f32: l_out-13 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -156.92, avg = 0.03, max = 308.02, var = 17.31 | |
| 0.40.504.653 I 00575: f32: norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.06, avg = 0.01, max = 65.61, var = 1.00 | |
| 0.40.524.231 I 00576: f32: attn_norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.95, avg = 0.00, max = 58.99, var = 0.39 | |
| 0.40.591.545 I 00577: f32: Qcur-14 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.37, avg = -0.04, max = 31.06, var = 2.25 | |
| 0.40.621.739 I 00578: f32: Qcur-14 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.37, avg = -0.04, max = 31.06, var = 2.25 | |
| 0.40.652.084 I 00579: f32: norm-14 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.74, avg = -0.03, max = 8.29, var = 1.00 | |
| 0.40.681.471 I 00580: f32: Qcur_normed-14 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.04, avg = -0.03, max = 8.48, var = 1.05 | |
| 0.40.711.312 I 00581: f32: Qcur_pos-14 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.64, avg = -0.02, max = 11.63, var = 1.05 | |
| 0.40.745.298 I 00582: f32: Kcur-14 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.08, avg = -0.04, max = 96.69, var = 3.58 | |
| 0.40.760.415 I 00583: f32: Kcur-14 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.08, avg = -0.04, max = 96.69, var = 3.58 | |
| 0.40.775.809 I 00584: f32: norm-14 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.85, avg = -0.02, max = 12.81, var = 1.00 | |
| 0.40.790.769 I 00585: f32: Kcur_normed-14 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.56, var = 0.01 | |
| 0.40.805.925 I 00586: f32: Kcur_pos-14 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.55, var = 0.01 | |
| 0.40.839.819 I 00587: f32: Vcur-14 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -181.61, avg = 0.00, max = 330.80, var = 1.59 | |
| 0.40.854.927 I 00588: f32: Vcur-14 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -181.61, avg = 0.00, max = 330.80, var = 1.59 | |
| 0.40.870.354 I 00589: f32: Vcur_normed-14 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.40.885.040 I 00590: f32: Kcur_pos-14 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.55, var = 0.01 | |
| 0.40.929.080 I 00591: f16: cache_k_l14 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.55, var = 0.00 | |
| 0.40.943.741 I 00592: f32: Vcur_normed-14 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.40.987.665 I 00593: f16: cache_v_l14 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 0.33 | |
| 0.41.016.950 I 00594: f32: Qcur_pos-14 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.64, avg = -0.02, max = 11.63, var = 1.05 | |
| 0.41.046.203 I 00595: f32: Qcur_pos-14 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.64, avg = -0.02, max = 11.63, var = 1.05 | |
| 0.41.060.812 I 00596: f16: cache_k_l14 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.55, var = 0.01 | |
| 0.41.075.442 I 00597: f16: cache_k_l14 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.55, var = 0.01 | |
| 0.41.090.002 I 00598: f16: cache_v_l14 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.41.104.549 I 00599: f16: cache_v_l14 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.41.144.955 I 00600: f32: __fattn__-14 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 0.43 | |
| 0.41.175.180 I 00601: f32: kqv_out-14 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 16.00, var = 0.43 | |
| 0.41.232.348 I 00602: f32: node_601 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.55, avg = 0.00, max = 94.78, var = 3.64 | |
| 0.41.252.210 I 00603: f32: norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.31, avg = 0.00, max = 32.59, var = 1.00 | |
| 0.41.271.808 I 00604: f32: attn_post_norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -2.23, avg = -0.00, max = 3.77, var = 0.00 | |
| 0.41.291.598 I 00605: f32: attn_out-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -158.82, avg = 0.03, max = 310.64, var = 17.67 | |
| 0.41.311.517 I 00606: f32: norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.10, avg = 0.01, max = 65.57, var = 1.00 | |
| 0.41.331.036 I 00607: f32: ffn_norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -159.77, avg = -0.01, max = 55.76, var = 0.72 | |
| 0.41.506.925 I 00608: f32: ffn_gate-14 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.41, avg = -2.40, max = 50.85, var = 1.78 | |
| 0.41.681.500 I 00609: f32: ffn_up-14 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -51.43, avg = 0.00, max = 56.27, var = 1.87 | |
| 0.41.762.662 I 00610: f32: ffn_geglu-14 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -813.75, avg = -0.00, max = 1234.08, var = 0.54 | |
| 0.41.879.863 I 00611: f32: ffn_out-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -709.13, avg = -0.02, max = 786.72, var = 12.69 | |
| 0.41.899.788 I 00612: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.14, avg = -0.01, max = 29.34, var = 1.00 | |
| 0.41.919.455 I 00613: f32: ffn_post_norm-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.56, avg = 0.00, max = 11.23, var = 0.04 | |
| 0.41.939.202 I 00614: f32: node_613 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -164.60, avg = 0.03, max = 321.02, var = 18.80 | |
| 0.41.959.479 I 00615: f32: l_out-14 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -161.39, avg = 0.03, max = 314.75, var = 18.07 | |
| 0.41.979.345 I 00616: f32: norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.35, avg = 0.01, max = 65.45, var = 1.00 | |
| 0.41.999.028 I 00617: f32: attn_norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -86.01, avg = 0.01, max = 150.68, var = 2.02 | |
| 0.42.066.519 I 00618: f32: Qcur-15 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -86.03, avg = -0.02, max = 113.70, var = 19.89 | |
| 0.42.096.711 I 00619: f32: Qcur-15 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -86.03, avg = -0.02, max = 113.70, var = 19.89 | |
| 0.42.127.091 I 00620: f32: norm-15 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.40, avg = -0.01, max = 12.10, var = 1.00 | |
| 0.42.156.823 I 00621: f32: Qcur_normed-15 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.75, avg = -0.01, max = 12.48, var = 1.06 | |
| 0.42.186.737 I 00622: f32: Qcur_pos-15 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.56, avg = -0.00, max = 12.26, var = 1.06 | |
| 0.42.220.970 I 00623: f32: Kcur-15 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -128.76, avg = 0.05, max = 105.34, var = 30.75 | |
| 0.42.236.079 I 00624: f32: Kcur-15 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -128.76, avg = 0.05, max = 105.34, var = 30.75 | |
| 0.42.251.231 I 00625: f32: norm-15 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.95, avg = 0.01, max = 9.78, var = 1.00 | |
| 0.42.266.197 I 00626: f32: Kcur_normed-15 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.08, avg = 0.00, max = 1.18, var = 0.01 | |
| 0.42.281.360 I 00627: f32: Kcur_pos-15 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.08, avg = -0.00, max = 1.17, var = 0.01 | |
| 0.42.315.407 I 00628: f32: Vcur-15 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -454.97, avg = -0.01, max = 466.50, var = 8.25 | |
| 0.42.330.501 I 00629: f32: Vcur-15 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -454.97, avg = -0.01, max = 466.50, var = 8.25 | |
| 0.42.346.140 I 00630: f32: Vcur_normed-15 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.79, var = 1.00 | |
| 0.42.361.278 I 00631: f32: Kcur_pos-15 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.08, avg = -0.00, max = 1.17, var = 0.01 | |
| 0.42.405.338 I 00632: f16: cache_k_l15 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.08, avg = -0.00, max = 1.17, var = 0.00 | |
| 0.42.419.994 I 00633: f32: Vcur_normed-15 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.79, var = 1.00 | |
| 0.42.463.915 I 00634: f16: cache_v_l15 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.79, var = 0.33 | |
| 0.42.493.191 I 00635: f32: Qcur_pos-15 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.56, avg = -0.00, max = 12.26, var = 1.06 | |
| 0.42.522.476 I 00636: f32: Qcur_pos-15 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.56, avg = -0.00, max = 12.26, var = 1.06 | |
| 0.42.537.080 I 00637: f16: cache_k_l15 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.08, avg = -0.00, max = 1.17, var = 0.01 | |
| 0.42.551.686 I 00638: f16: cache_k_l15 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.08, avg = -0.00, max = 1.17, var = 0.01 | |
| 0.42.566.243 I 00639: f16: cache_v_l15 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.79, var = 1.00 | |
| 0.42.580.800 I 00640: f16: cache_v_l15 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.79, var = 1.00 | |
| 0.42.620.459 I 00641: f32: __fattn__-15 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.79, var = 0.68 | |
| 0.42.649.756 I 00642: f32: kqv_out-15 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.79, var = 0.68 | |
| 0.42.706.802 I 00643: f32: node_642 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -94.49, avg = 0.04, max = 215.26, var = 8.14 | |
| 0.42.727.222 I 00644: f32: norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.81, avg = 0.01, max = 61.87, var = 1.00 | |
| 0.42.746.605 I 00645: f32: attn_post_norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.07, avg = 0.01, max = 69.12, var = 0.94 | |
| 0.42.766.355 I 00646: f32: attn_out-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -191.84, avg = 0.05, max = 381.53, var = 26.89 | |
| 0.42.786.302 I 00647: f32: norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.09, avg = 0.01, max = 65.98, var = 1.00 | |
| 0.42.805.782 I 00648: f32: ffn_norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -114.95, avg = -0.00, max = 48.45, var = 0.35 | |
| 0.42.981.706 I 00649: f32: ffn_gate-15 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.53, avg = -1.68, max = 35.71, var = 1.35 | |
| 0.43.157.351 I 00650: f32: ffn_up-15 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -99.25, avg = -0.00, max = 93.30, var = 1.61 | |
| 0.43.238.423 I 00651: f32: ffn_geglu-15 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -214.78, avg = 0.00, max = 1462.37, var = 0.35 | |
| 0.43.355.280 I 00652: f32: ffn_out-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -447.33, avg = -0.00, max = 503.41, var = 3.18 | |
| 0.43.375.229 I 00653: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.07, avg = -0.00, max = 49.30, var = 1.00 | |
| 0.43.394.865 I 00654: f32: ffn_post_norm-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.43, avg = 0.00, max = 37.74, var = 0.27 | |
| 0.43.414.664 I 00655: f32: node_654 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -208.80, avg = 0.05, max = 413.77, var = 31.77 | |
| 0.43.434.748 I 00656: f32: l_out-15 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -172.92, avg = 0.04, max = 342.65, var = 21.79 | |
| 0.43.454.782 I 00657: f32: norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.82, avg = 0.01, max = 65.86, var = 1.00 | |
| 0.43.474.381 I 00658: f32: attn_norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.93, avg = 0.00, max = 67.83, var = 0.15 | |
| 0.43.541.807 I 00659: f32: Qcur-16 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.22, avg = 0.04, max = 28.55, var = 1.32 | |
| 0.43.572.009 I 00660: f32: Qcur-16 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.22, avg = 0.04, max = 28.55, var = 1.32 | |
| 0.43.603.034 I 00661: f32: norm-16 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.73, avg = 0.03, max = 13.63, var = 1.00 | |
| 0.43.632.684 I 00662: f32: Qcur_normed-16 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.13, avg = 0.03, max = 14.06, var = 1.06 | |
| 0.43.662.608 I 00663: f32: Qcur_pos-16 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.12, avg = 0.01, max = 14.05, var = 1.06 | |
| 0.43.696.506 I 00664: f32: Kcur-16 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.28, avg = 0.04, max = 83.87, var = 1.77 | |
| 0.43.711.613 I 00665: f32: Kcur-16 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.28, avg = 0.04, max = 83.87, var = 1.77 | |
| 0.43.727.269 I 00666: f32: norm-16 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.33, avg = 0.03, max = 11.95, var = 1.00 | |
| 0.43.742.324 I 00667: f32: Kcur_normed-16 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.89, avg = 0.00, max = 1.45, var = 0.01 | |
| 0.43.757.485 I 00668: f32: Kcur_pos-16 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = 0.00, max = 1.45, var = 0.01 | |
| 0.43.791.374 I 00669: f32: Vcur-16 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -140.74, avg = -0.00, max = 215.40, var = 0.58 | |
| 0.43.806.486 I 00670: f32: Vcur-16 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -140.74, avg = -0.00, max = 215.40, var = 0.58 | |
| 0.43.822.094 I 00671: f32: Vcur_normed-16 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.43.837.228 I 00672: f32: Kcur_pos-16 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = 0.00, max = 1.45, var = 0.01 | |
| 0.43.881.301 I 00673: f16: cache_k_l16 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = 0.00, max = 1.45, var = 0.00 | |
| 0.43.895.958 I 00674: f32: Vcur_normed-16 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.43.939.859 I 00675: f16: cache_v_l16 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 0.33 | |
| 0.43.969.129 I 00676: f32: Qcur_pos-16 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.12, avg = 0.01, max = 14.05, var = 1.06 | |
| 0.43.998.387 I 00677: f32: Qcur_pos-16 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.12, avg = 0.01, max = 14.05, var = 1.06 | |
| 0.44.013.009 I 00678: f16: cache_k_l16 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = 0.00, max = 1.45, var = 0.01 | |
| 0.44.027.616 I 00679: f16: cache_k_l16 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -0.88, avg = 0.00, max = 1.45, var = 0.01 | |
| 0.44.042.162 I 00680: f16: cache_v_l16 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.44.056.713 I 00681: f16: cache_v_l16 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.44.096.608 I 00682: f32: __fattn__-16 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 0.34 | |
| 0.44.125.905 I 00683: f32: kqv_out-16 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 0.34 | |
| 0.44.183.153 I 00684: f32: node_683 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -42.72, avg = 0.01, max = 55.81, var = 2.07 | |
| 0.44.203.342 I 00685: f32: norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.69, avg = 0.01, max = 29.32, var = 1.00 | |
| 0.44.222.914 I 00686: f32: attn_post_norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -2.90, avg = 0.00, max = 5.10, var = 0.01 | |
| 0.44.242.679 I 00687: f32: attn_out-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -175.55, avg = 0.04, max = 347.00, var = 22.38 | |
| 0.44.262.635 I 00688: f32: norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.92, avg = 0.01, max = 65.80, var = 1.00 | |
| 0.44.282.151 I 00689: f32: ffn_norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -67.64, avg = 0.00, max = 213.55, var = 0.63 | |
| 0.44.457.995 I 00690: f32: ffn_gate-16 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.75, avg = -1.80, max = 21.04, var = 1.41 | |
| 0.44.634.056 I 00691: f32: ffn_up-16 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.90, avg = -0.00, max = 120.82, var = 1.22 | |
| 0.44.715.109 I 00692: f32: ffn_geglu-16 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -843.29, avg = 0.00, max = 127.37, var = 0.19 | |
| 0.44.831.839 I 00693: f32: ffn_out-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -171.83, avg = -0.01, max = 208.26, var = 1.86 | |
| 0.44.852.023 I 00694: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.27, avg = -0.01, max = 33.91, var = 1.00 | |
| 0.44.871.677 I 00695: f32: ffn_post_norm-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -5.94, avg = 0.00, max = 12.39, var = 0.05 | |
| 0.44.891.262 I 00696: f32: node_695 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -180.36, avg = 0.04, max = 357.15, var = 23.76 | |
| 0.44.911.294 I 00697: f32: l_out-16 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -176.83, avg = 0.04, max = 350.18, var = 22.84 | |
| 0.44.931.332 I 00698: f32: norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.03, avg = 0.01, max = 65.64, var = 1.00 | |
| 0.44.950.876 I 00699: f32: attn_norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -118.45, avg = 0.01, max = 376.50, var = 3.04 | |
| 0.45.085.538 I 00700: f32: Qcur-17 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -120.74, avg = 0.00, max = 89.55, var = 12.84 | |
| 0.45.144.688 I 00701: f32: Qcur-17 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -120.74, avg = 0.00, max = 89.55, var = 12.84 | |
| 0.45.205.557 I 00702: f32: norm-17 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.44, avg = -0.00, max = 14.36, var = 1.00 | |
| 0.45.264.922 I 00703: f32: Qcur_normed-17 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.86, avg = -0.00, max = 14.81, var = 1.06 | |
| 0.45.324.638 I 00704: f32: Qcur_pos-17 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.36, avg = 0.00, max = 16.58, var = 1.06 | |
| 0.45.342.107 I 00705: f32: Vcur-17 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -325.56, avg = -0.02, max = 419.21, var = 7.86 | |
| 0.45.349.656 I 00706: f32: Vcur-17 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -325.56, avg = -0.02, max = 419.21, var = 7.86 | |
| 0.45.357.536 I 00707: f32: norm-17 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = -0.01, max = 22.33, var = 1.00 | |
| 0.45.365.006 I 00708: f32: Kcur_normed-17 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.35, var = 0.00 | |
| 0.45.372.591 I 00709: f32: Kcur_pos-17 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.35, var = 0.00 | |
| 0.45.379.931 I 00710: f32: Vcur-17 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -325.56, avg = -0.02, max = 419.21, var = 7.86 | |
| 0.45.388.010 I 00711: f32: Vcur_normed-17 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = -0.01, max = 22.33, var = 1.00 | |
| 0.45.395.381 I 00712: f32: Kcur_pos-17 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.35, var = 0.00 | |
| 0.45.453.714 I 00713: f16: cache_k_l17 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.35, var = 0.00 | |
| 0.45.461.042 I 00714: f32: Vcur_normed-17 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = -0.01, max = 22.33, var = 1.00 | |
| 0.45.519.379 I 00715: f16: cache_v_l17 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = -0.00, max = 22.33, var = 0.12 | |
| 0.45.577.867 I 00716: f32: Qcur_pos-17 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.36, avg = 0.00, max = 16.58, var = 1.06 | |
| 0.45.636.308 I 00717: f32: Qcur_pos-17 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.36, avg = 0.00, max = 16.58, var = 1.06 | |
| 0.45.643.638 I 00718: f16: cache_k_l17 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.35, var = 0.00 | |
| 0.45.650.996 I 00719: f16: cache_k_l17 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.35, var = 0.00 | |
| 0.45.658.286 I 00720: f16: cache_v_l17 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = -0.01, max = 22.33, var = 1.00 | |
| 0.45.665.571 I 00721: f16: cache_v_l17 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = -0.01, max = 22.33, var = 1.00 | |
| 0.45.744.692 I 00722: f32: __fattn__-17 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = 0.01, max = 22.33, var = 0.60 | |
| 0.45.805.149 I 00723: f32: kqv_out-17 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.28, avg = 0.01, max = 22.33, var = 0.60 | |
| 0.45.898.646 I 00724: f32: node_723 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -92.33, avg = 0.05, max = 228.30, var = 9.97 | |
| 0.45.918.982 I 00725: f32: norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.73, avg = 0.02, max = 63.79, var = 1.00 | |
| 0.45.938.664 I 00726: f32: attn_post_norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.96, avg = 0.02, max = 83.22, var = 1.38 | |
| 0.45.958.441 I 00727: f32: attn_out-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -205.42, avg = 0.06, max = 432.66, var = 34.87 | |
| 0.45.978.383 I 00728: f32: norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.67, avg = 0.01, max = 66.20, var = 1.00 | |
| 0.45.997.887 I 00729: f32: ffn_norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.10, avg = -0.00, max = 197.72, var = 0.92 | |
| 0.46.173.427 I 00730: f32: ffn_gate-17 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.82, avg = -2.22, max = 26.13, var = 1.51 | |
| 0.46.349.170 I 00731: f32: ffn_up-17 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -45.91, avg = -0.00, max = 59.06, var = 1.28 | |
| 0.46.430.247 I 00732: f32: ffn_geglu-17 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -771.13, avg = 0.00, max = 909.19, var = 0.29 | |
| 0.46.547.150 I 00733: f32: ffn_out-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -244.19, avg = 0.00, max = 261.07, var = 2.38 | |
| 0.46.567.057 I 00734: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.53, avg = 0.00, max = 29.69, var = 1.00 | |
| 0.46.586.701 I 00735: f32: ffn_post_norm-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.09, avg = 0.00, max = 14.50, var = 0.08 | |
| 0.46.606.434 I 00736: f32: node_735 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -210.05, avg = 0.06, max = 445.64, var = 36.91 | |
| 0.46.626.474 I 00737: f32: l_out-17 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -172.31, avg = 0.05, max = 365.56, var = 24.83 | |
| 0.46.646.549 I 00738: f32: norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.76, avg = 0.01, max = 66.04, var = 1.00 | |
| 0.46.666.140 I 00739: f32: attn_norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -77.35, avg = 0.01, max = 70.80, var = 0.38 | |
| 0.46.733.482 I 00740: f32: Qcur-18 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -52.32, avg = 0.01, max = 35.40, var = 2.33 | |
| 0.46.763.685 I 00741: f32: Qcur-18 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -52.32, avg = 0.01, max = 35.40, var = 2.33 | |
| 0.46.794.101 I 00742: f32: norm-18 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.61, avg = 0.00, max = 12.14, var = 1.00 | |
| 0.46.823.697 I 00743: f32: Qcur_normed-18 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.90, avg = 0.00, max = 12.43, var = 1.05 | |
| 0.46.853.638 I 00744: f32: Qcur_pos-18 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.49, avg = 0.00, max = 12.51, var = 1.05 | |
| 0.46.888.217 I 00745: f32: Kcur-18 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -110.46, avg = 0.01, max = 107.77, var = 3.20 | |
| 0.46.903.313 I 00746: f32: Kcur-18 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -110.46, avg = 0.01, max = 107.77, var = 3.20 | |
| 0.46.918.660 I 00747: f32: norm-18 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.70, avg = 0.00, max = 9.96, var = 1.00 | |
| 0.46.933.660 I 00748: f32: Kcur_normed-18 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = 0.00, max = 1.22, var = 0.01 | |
| 0.46.948.804 I 00749: f32: Kcur_pos-18 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.22, var = 0.01 | |
| 0.46.982.684 I 00750: f32: Vcur-18 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -286.92, avg = 0.00, max = 316.59, var = 1.41 | |
| 0.46.997.795 I 00751: f32: Vcur-18 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -286.92, avg = 0.00, max = 316.59, var = 1.41 | |
| 0.47.013.358 I 00752: f32: Vcur_normed-18 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.47.028.081 I 00753: f32: Kcur_pos-18 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.22, var = 0.01 | |
| 0.47.072.119 I 00754: f16: cache_k_l18 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.22, var = 0.00 | |
| 0.47.086.814 I 00755: f32: Vcur_normed-18 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.47.130.701 I 00756: f16: cache_v_l18 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.99, var = 0.33 | |
| 0.47.159.971 I 00757: f32: Qcur_pos-18 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.49, avg = 0.00, max = 12.51, var = 1.05 | |
| 0.47.189.261 I 00758: f32: Qcur_pos-18 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.49, avg = 0.00, max = 12.51, var = 1.05 | |
| 0.47.203.893 I 00759: f16: cache_k_l18 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.22, var = 0.01 | |
| 0.47.218.503 I 00760: f16: cache_k_l18 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.22, var = 0.01 | |
| 0.47.233.058 I 00761: f16: cache_v_l18 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.47.247.593 I 00762: f16: cache_v_l18 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.47.287.744 I 00763: f32: __fattn__-18 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.99, var = 0.38 | |
| 0.47.317.094 I 00764: f32: kqv_out-18 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.99, var = 0.38 | |
| 0.47.374.376 I 00765: f32: node_764 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.74, avg = 0.01, max = 69.48, var = 1.54 | |
| 0.47.394.743 I 00766: f32: norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.13, avg = 0.01, max = 43.52, var = 1.00 | |
| 0.47.414.337 I 00767: f32: attn_post_norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -4.99, avg = 0.00, max = 10.79, var = 0.03 | |
| 0.47.434.066 I 00768: f32: attn_out-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -176.60, avg = 0.05, max = 374.77, var = 26.04 | |
| 0.47.454.079 I 00769: f32: norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.87, avg = 0.01, max = 65.96, var = 1.00 | |
| 0.47.473.630 I 00770: f32: ffn_norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -190.00, avg = -0.01, max = 84.97, var = 0.98 | |
| 0.47.649.162 I 00771: f32: ffn_gate-18 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.75, avg = -2.59, max = 41.49, var = 1.96 | |
| 0.47.825.132 I 00772: f32: ffn_up-18 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.33, avg = 0.00, max = 48.25, var = 1.81 | |
| 0.47.906.307 I 00773: f32: ffn_geglu-18 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -201.96, avg = 0.00, max = 2002.14, var = 0.63 | |
| 0.48.023.183 I 00774: f32: ffn_out-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -302.03, avg = -0.01, max = 492.56, var = 7.86 | |
| 0.48.043.081 I 00775: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.07, avg = -0.01, max = 26.23, var = 1.00 | |
| 0.48.062.732 I 00776: f32: ffn_post_norm-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -5.51, avg = 0.00, max = 11.06, var = 0.06 | |
| 0.48.082.466 I 00777: f32: node_776 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -180.82, avg = 0.05, max = 384.10, var = 27.46 | |
| 0.48.102.500 I 00778: f32: l_out-18 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -175.88, avg = 0.05, max = 373.60, var = 25.97 | |
| 0.48.122.465 I 00779: f32: norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.94, avg = 0.01, max = 65.85, var = 1.00 | |
| 0.48.142.004 I 00780: f32: attn_norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -157.29, avg = 0.02, max = 147.81, var = 2.07 | |
| 0.48.209.441 I 00781: f32: Qcur-19 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -90.71, avg = 0.02, max = 85.05, var = 12.62 | |
| 0.48.239.636 I 00782: f32: Qcur-19 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -90.71, avg = 0.02, max = 85.05, var = 12.62 | |
| 0.48.270.011 I 00783: f32: norm-19 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.32, avg = 0.01, max = 12.30, var = 1.00 | |
| 0.48.299.634 I 00784: f32: Qcur_normed-19 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.32, avg = 0.01, max = 12.30, var = 1.00 | |
| 0.48.329.670 I 00785: f32: Qcur_pos-19 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = 0.01, max = 11.78, var = 1.00 | |
| 0.48.364.062 I 00786: f32: Kcur-19 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -300.01, avg = -0.06, max = 217.01, var = 15.92 | |
| 0.48.379.155 I 00787: f32: Kcur-19 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -300.01, avg = -0.06, max = 217.01, var = 15.92 | |
| 0.48.394.492 I 00788: f32: norm-19 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.28, avg = -0.02, max = 10.77, var = 1.00 | |
| 0.48.409.461 I 00789: f32: Kcur_normed-19 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.79, avg = -0.00, max = 1.35, var = 0.02 | |
| 0.48.424.612 I 00790: f32: Kcur_pos-19 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.78, avg = -0.00, max = 1.35, var = 0.02 | |
| 0.48.458.580 I 00791: f32: Vcur-19 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -702.25, avg = 0.02, max = 911.49, var = 9.53 | |
| 0.48.473.690 I 00792: f32: Vcur-19 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -702.25, avg = 0.02, max = 911.49, var = 9.53 | |
| 0.48.489.030 I 00793: f32: Vcur_normed-19 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.48.503.708 I 00794: f32: Kcur_pos-19 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.78, avg = -0.00, max = 1.35, var = 0.02 | |
| 0.48.547.747 I 00795: f16: cache_k_l19 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.78, avg = -0.00, max = 1.35, var = 0.01 | |
| 0.48.562.398 I 00796: f32: Vcur_normed-19 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.48.606.356 I 00797: f16: cache_v_l19 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 0.33 | |
| 0.48.635.645 I 00798: f32: Qcur_pos-19 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = 0.01, max = 11.78, var = 1.00 | |
| 0.48.664.913 I 00799: f32: Qcur_pos-19 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = 0.01, max = 11.78, var = 1.00 | |
| 0.48.679.520 I 00800: f16: cache_k_l19 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.78, avg = -0.00, max = 1.35, var = 0.02 | |
| 0.48.694.122 I 00801: f16: cache_k_l19 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.78, avg = -0.00, max = 1.35, var = 0.02 | |
| 0.48.708.684 I 00802: f16: cache_v_l19 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.48.723.231 I 00803: f16: cache_v_l19 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 0.48.763.002 I 00804: f32: __fattn__-19 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 16.00, var = 0.53 | |
| 0.48.792.323 I 00805: f32: kqv_out-19 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 16.00, var = 0.53 | |
| 0.48.849.075 I 00806: f32: node_805 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -79.81, avg = -0.00, max = 171.34, var = 6.34 | |
| 0.48.869.421 I 00807: f32: norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.75, avg = -0.00, max = 59.27, var = 1.00 | |
| 0.48.888.793 I 00808: f32: attn_post_norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.20, avg = 0.00, max = 47.93, var = 0.35 | |
| 0.48.908.672 I 00809: f32: attn_out-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -191.61, avg = 0.06, max = 415.80, var = 31.86 | |
| 0.48.928.629 I 00810: f32: norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.63, avg = 0.01, max = 66.11, var = 1.00 | |
| 0.48.948.120 I 00811: f32: ffn_norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.59, avg = 0.00, max = 62.92, var = 0.27 | |
| 0.49.123.774 I 00812: f32: ffn_gate-19 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.53, avg = -1.10, max = 61.62, var = 0.97 | |
| 0.49.297.723 I 00813: f32: ffn_up-19 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.30, avg = 0.00, max = 22.34, var = 0.66 | |
| 0.49.378.924 I 00814: f32: ffn_geglu-19 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1928.84, avg = -0.00, max = 181.11, var = 0.42 | |
| 0.49.495.721 I 00815: f32: ffn_out-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -371.60, avg = 0.01, max = 349.92, var = 3.93 | |
| 0.49.515.626 I 00816: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.04, avg = 0.01, max = 37.50, var = 1.00 | |
| 0.49.535.239 I 00817: f32: ffn_post_norm-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.87, avg = 0.00, max = 22.12, var = 0.14 | |
| 0.49.554.958 I 00818: f32: node_817 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -200.12, avg = 0.06, max = 436.30, var = 34.93 | |
| 0.49.575.020 I 00819: f32: l_out-19 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -179.02, avg = 0.05, max = 390.29, var = 27.95 | |
| 0.49.594.839 I 00820: f32: norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.54, avg = 0.01, max = 66.11, var = 1.00 | |
| 0.49.614.418 I 00821: f32: attn_norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -101.11, avg = 0.01, max = 140.89, var = 2.19 | |
| 0.49.682.174 I 00822: f32: Qcur-20 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.53, avg = -0.03, max = 68.59, var = 13.16 | |
| 0.49.712.153 I 00823: f32: Qcur-20 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.53, avg = -0.03, max = 68.59, var = 13.16 | |
| 0.49.742.417 I 00824: f32: norm-20 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.97, avg = -0.01, max = 9.50, var = 1.00 | |
| 0.49.772.180 I 00825: f32: Qcur_normed-20 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.61, avg = -0.01, max = 9.24, var = 0.95 | |
| 0.49.802.192 I 00826: f32: Qcur_pos-20 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.52, avg = -0.01, max = 8.98, var = 0.95 | |
| 0.49.836.292 I 00827: f32: Kcur-20 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -170.39, avg = -0.04, max = 226.74, var = 21.50 | |
| 0.49.851.405 I 00828: f32: Kcur-20 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -170.39, avg = -0.04, max = 226.74, var = 21.50 | |
| 0.49.866.750 I 00829: f32: norm-20 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.69, avg = -0.01, max = 9.99, var = 1.00 | |
| 0.49.881.835 I 00830: f32: Kcur_normed-20 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.25, avg = -0.00, max = 1.29, var = 0.02 | |
| 0.49.896.779 I 00831: f32: Kcur_pos-20 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.25, avg = -0.00, max = 1.29, var = 0.02 | |
| 0.49.931.108 I 00832: f32: Vcur-20 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -428.47, avg = 0.02, max = 713.24, var = 6.25 | |
| 0.49.946.203 I 00833: f32: Vcur-20 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -428.47, avg = 0.02, max = 713.24, var = 6.25 | |
| 0.49.961.532 I 00834: f32: Vcur_normed-20 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.49.976.218 I 00835: f32: Kcur_pos-20 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.25, avg = -0.00, max = 1.29, var = 0.02 | |
| 0.50.020.292 I 00836: f16: cache_k_l20 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.25, avg = -0.00, max = 1.29, var = 0.01 | |
| 0.50.034.955 I 00837: f32: Vcur_normed-20 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.50.078.833 I 00838: f16: cache_v_l20 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 0.33 | |
| 0.50.108.152 I 00839: f32: Qcur_pos-20 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.52, avg = -0.01, max = 8.98, var = 0.95 | |
| 0.50.137.431 I 00840: f32: Qcur_pos-20 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.52, avg = -0.01, max = 8.98, var = 0.95 | |
| 0.50.152.061 I 00841: f16: cache_k_l20 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.25, avg = -0.00, max = 1.29, var = 0.02 | |
| 0.50.166.668 I 00842: f16: cache_k_l20 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.25, avg = -0.00, max = 1.29, var = 0.02 | |
| 0.50.181.216 I 00843: f16: cache_v_l20 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.50.195.765 I 00844: f16: cache_v_l20 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 16.00, var = 1.00 | |
| 0.50.235.634 I 00845: f32: __fattn__-20 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.02, max = 16.00, var = 0.58 | |
| 0.50.264.953 I 00846: f32: kqv_out-20 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.02, max = 16.00, var = 0.58 | |
| 0.50.322.082 I 00847: f32: node_846 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -75.74, avg = 0.03, max = 184.55, var = 7.97 | |
| 0.50.342.427 I 00848: f32: norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.70, avg = 0.01, max = 58.38, var = 1.00 | |
| 0.50.362.043 I 00849: f32: attn_post_norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.43, avg = 0.00, max = 45.38, var = 0.35 | |
| 0.50.381.755 I 00850: f32: attn_out-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -196.54, avg = 0.06, max = 434.81, var = 33.95 | |
| 0.50.401.815 I 00851: f32: norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.58, avg = 0.01, max = 66.34, var = 1.00 | |
| 0.50.421.366 I 00852: f32: ffn_norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.12, avg = -0.00, max = 51.96, var = 0.29 | |
| 0.50.597.041 I 00853: f32: ffn_gate-20 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.02, avg = -1.21, max = 15.36, var = 1.05 | |
| 0.50.773.218 I 00854: f32: ffn_up-20 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -91.69, avg = 0.01, max = 17.56, var = 0.68 | |
| 0.50.855.272 I 00855: f32: ffn_geglu-20 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -421.82, avg = 0.00, max = 115.76, var = 0.07 | |
| 0.50.972.044 I 00856: f32: ffn_out-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -221.78, avg = 0.01, max = 223.41, var = 1.87 | |
| 0.50.992.434 I 00857: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.52, avg = 0.01, max = 44.30, var = 1.00 | |
| 0.51.012.074 I 00858: f32: ffn_post_norm-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.52, avg = 0.00, max = 28.21, var = 0.19 | |
| 0.51.031.864 I 00859: f32: node_858 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -205.37, avg = 0.06, max = 460.96, var = 38.05 | |
| 0.51.052.315 I 00860: f32: l_out-20 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -183.71, avg = 0.05, max = 412.34, var = 30.45 | |
| 0.51.072.352 I 00861: f32: norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.10, avg = 0.01, max = 66.34, var = 1.00 | |
| 0.51.091.914 I 00862: f32: attn_norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.90, avg = 0.00, max = 92.32, var = 0.64 | |
| 0.51.167.043 I 00863: f32: Qcur-21 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -82.19, avg = -0.00, max = 49.10, var = 3.98 | |
| 0.51.197.233 I 00864: f32: Qcur-21 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -82.19, avg = -0.00, max = 49.10, var = 3.98 | |
| 0.51.227.736 I 00865: f32: norm-21 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.87, avg = -0.00, max = 12.22, var = 1.00 | |
| 0.51.257.371 I 00866: f32: Qcur_normed-21 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.82, avg = -0.00, max = 12.18, var = 0.99 | |
| 0.51.287.269 I 00867: f32: Qcur_pos-21 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.80, avg = -0.01, max = 11.33, var = 0.99 | |
| 0.51.321.758 I 00868: f32: Kcur-21 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -148.19, avg = -0.01, max = 126.92, var = 6.27 | |
| 0.51.336.866 I 00869: f32: Kcur-21 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -148.19, avg = -0.01, max = 126.92, var = 6.27 | |
| 0.51.352.241 I 00870: f32: norm-21 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.36, avg = -0.01, max = 11.42, var = 1.00 | |
| 0.51.367.239 I 00871: f32: Kcur_normed-21 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.44, var = 0.02 | |
| 0.51.382.390 I 00872: f32: Kcur_pos-21 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.44, var = 0.02 | |
| 0.51.416.022 I 00873: f32: Vcur-21 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -346.13, avg = 0.00, max = 292.07, var = 2.18 | |
| 0.51.431.130 I 00874: f32: Vcur-21 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -346.13, avg = 0.00, max = 292.07, var = 2.18 | |
| 0.51.446.708 I 00875: f32: Vcur_normed-21 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.51.461.529 I 00876: f32: Kcur_pos-21 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.44, var = 0.02 | |
| 0.51.505.585 I 00877: f16: cache_k_l21 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.44, var = 0.01 | |
| 0.51.520.241 I 00878: f32: Vcur_normed-21 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 0.51.564.637 I 00879: f16: cache_v_l21 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 0.33 | |
| 0.51.594.737 I 00880: f32: Qcur_pos-21 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.80, avg = -0.01, max = 11.33, var = 0.99 | |
| 0.51.624.931 I 00881: f32: Qcur_pos-21 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.80, avg = -0.01, max = 11.33, var = 0.99 | |
| 0.51.639.619 I 00882: f16: cache_k_l21 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.44, var = 0.02 | |
| 0.51.654.220 I 00883: f16: cache_k_l21 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.29, avg = -0.00, max = 1.44, var = 0.02 | |
| 0.51.668.767 I 00884: f16: cache_v_l21 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.51.683.311 I 00885: f16: cache_v_l21 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 1.00 | |
| 0.51.724.061 I 00886: f32: __fattn__-21 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.90, var = 0.33 | |
| 0.51.754.282 I 00887: f32: kqv_out-21 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.90, var = 0.33 | |
| 0.51.811.383 I 00888: f32: node_887 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.99, avg = -0.00, max = 42.20, var = 1.08 | |
| 0.51.831.774 I 00889: f32: norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.25, avg = -0.00, max = 41.26, var = 1.00 | |
| 0.51.851.870 I 00890: f32: attn_post_norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -4.13, avg = 0.00, max = 12.49, var = 0.04 | |
| 0.51.872.127 I 00891: f32: attn_out-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -186.61, avg = 0.05, max = 424.83, var = 31.54 | |
| 0.51.891.939 I 00892: f32: norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.16, avg = 0.01, max = 66.37, var = 1.00 | |
| 0.51.911.736 I 00893: f32: ffn_norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -43.21, avg = -0.00, max = 49.57, var = 0.31 | |
| 0.52.087.362 I 00894: f32: ffn_gate-21 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.41, avg = -1.23, max = 13.39, var = 1.03 | |
| 0.52.263.188 I 00895: f32: ffn_up-21 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -89.60, avg = 0.00, max = 80.80, var = 0.90 | |
| 0.52.344.092 I 00896: f32: ffn_geglu-21 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -530.03, avg = 0.00, max = 259.17, var = 0.09 | |
| 0.52.460.795 I 00897: f32: ffn_out-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -356.27, avg = 0.01, max = 348.00, var = 2.83 | |
| 0.52.481.255 I 00898: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.74, avg = 0.02, max = 43.91, var = 1.00 | |
| 0.52.501.367 I 00899: f32: ffn_post_norm-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.90, avg = 0.00, max = 23.84, var = 0.15 | |
| 0.52.521.633 I 00900: f32: node_899 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -191.56, avg = 0.06, max = 447.54, var = 34.45 | |
| 0.52.541.999 I 00901: f32: l_out-21 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -182.58, avg = 0.05, max = 426.56, var = 31.30 | |
| 0.52.562.044 I 00902: f32: norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.62, avg = 0.01, max = 66.48, var = 1.00 | |
| 0.52.582.063 I 00903: f32: attn_norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.56, avg = 0.01, max = 118.12, var = 1.28 | |
| 0.52.649.417 I 00904: f32: Qcur-22 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -84.70, avg = 0.02, max = 89.68, var = 10.12 | |
| 0.52.679.616 I 00905: f32: Qcur-22 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -84.70, avg = 0.02, max = 89.68, var = 10.12 | |
| 0.52.710.733 I 00906: f32: norm-22 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.18, avg = 0.00, max = 13.50, var = 1.00 | |
| 0.52.741.212 I 00907: f32: Qcur_normed-22 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.95, avg = 0.00, max = 13.23, var = 0.96 | |
| 0.52.771.945 I 00908: f32: Qcur_pos-22 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.96, avg = 0.00, max = 13.23, var = 0.96 | |
| 0.52.805.672 I 00909: f32: Kcur-22 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -172.56, avg = 0.02, max = 168.88, var = 14.06 | |
| 0.52.820.786 I 00910: f32: Kcur-22 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -172.56, avg = 0.02, max = 168.88, var = 14.06 | |
| 0.52.836.452 I 00911: f32: norm-22 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.94, avg = 0.01, max = 10.46, var = 1.00 | |
| 0.52.851.832 I 00912: f32: Kcur_normed-22 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.27, avg = 0.00, max = 1.34, var = 0.02 | |
| 0.52.867.375 I 00913: f32: Kcur_pos-22 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.26, avg = 0.00, max = 1.34, var = 0.02 | |
| 0.52.901.368 I 00914: f32: Vcur-22 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -368.90, avg = 0.02, max = 427.25, var = 5.08 | |
| 0.52.916.473 I 00915: f32: Vcur-22 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -368.90, avg = 0.02, max = 427.25, var = 5.08 | |
| 0.52.932.133 I 00916: f32: Vcur_normed-22 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.01, max = 15.98, var = 1.00 | |
| 0.52.947.266 I 00917: f32: Kcur_pos-22 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.26, avg = 0.00, max = 1.34, var = 0.02 | |
| 0.52.991.265 I 00918: f16: cache_k_l22 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.26, avg = 0.00, max = 1.34, var = 0.01 | |
| 0.53.006.233 I 00919: f32: Vcur_normed-22 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.01, max = 15.98, var = 1.00 | |
| 0.53.050.071 I 00920: f16: cache_v_l22 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.98, var = 0.33 | |
| 0.53.080.062 I 00921: f32: Qcur_pos-22 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.96, avg = 0.00, max = 13.23, var = 0.96 | |
| 0.53.110.260 I 00922: f32: Qcur_pos-22 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.96, avg = 0.00, max = 13.23, var = 0.96 | |
| 0.53.124.947 I 00923: f16: cache_k_l22 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.26, avg = 0.00, max = 1.34, var = 0.02 | |
| 0.53.139.563 I 00924: f16: cache_k_l22 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.26, avg = 0.00, max = 1.34, var = 0.02 | |
| 0.53.154.109 I 00925: f16: cache_v_l22 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.01, max = 15.98, var = 1.00 | |
| 0.53.168.657 I 00926: f16: cache_v_l22 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.01, max = 15.98, var = 1.00 | |
| 0.53.209.131 I 00927: f32: __fattn__-22 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = 0.00, max = 15.98, var = 0.40 | |
| 0.53.239.359 I 00928: f32: kqv_out-22 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = 0.00, max = 15.98, var = 0.40 | |
| 0.53.296.341 I 00929: f32: node_928 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.28, avg = 0.01, max = 45.00, var = 1.01 | |
| 0.53.316.710 I 00930: f32: norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.09, avg = 0.01, max = 41.45, var = 1.00 | |
| 0.53.336.384 I 00931: f32: attn_post_norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.50, avg = 0.00, max = 24.29, var = 0.11 | |
| 0.53.356.630 I 00932: f32: attn_out-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -186.34, avg = 0.06, max = 444.01, var = 33.66 | |
| 0.53.376.734 I 00933: f32: norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.12, avg = 0.01, max = 66.78, var = 1.00 | |
| 0.53.396.832 I 00934: f32: ffn_norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.47, avg = -0.00, max = 33.43, var = 0.27 | |
| 0.53.572.292 I 00935: f32: ffn_gate-22 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.30, avg = -1.11, max = 26.72, var = 0.94 | |
| 0.53.747.959 I 00936: f32: ffn_up-22 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -70.15, avg = -0.00, max = 62.15, var = 0.67 | |
| 0.53.828.739 I 00937: f32: ffn_geglu-22 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -189.57, avg = 0.00, max = 516.41, var = 0.12 | |
| 0.53.945.519 I 00938: f32: ffn_out-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -311.31, avg = 0.01, max = 303.84, var = 1.81 | |
| 0.53.965.885 I 00939: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.82, avg = 0.01, max = 36.49, var = 1.00 | |
| 0.53.986.002 I 00940: f32: ffn_post_norm-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.17, avg = 0.01, max = 28.08, var = 0.28 | |
| 0.54.006.251 I 00941: f32: node_940 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -188.49, avg = 0.06, max = 468.08, var = 37.48 | |
| 0.54.026.685 I 00942: f32: l_out-22 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -170.08, avg = 0.06, max = 422.37, var = 30.52 | |
| 0.54.047.167 I 00943: f32: norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.87, avg = 0.01, max = 66.92, var = 1.00 | |
| 0.54.067.240 I 00944: f32: attn_norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.53, avg = -0.00, max = 168.41, var = 3.30 | |
| 0.54.201.496 I 00945: f32: Qcur-23 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -129.83, avg = -0.04, max = 168.13, var = 13.04 | |
| 0.54.261.878 I 00946: f32: Qcur-23 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -129.83, avg = -0.04, max = 168.13, var = 13.04 | |
| 0.54.324.436 I 00947: f32: norm-23 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.48, avg = -0.01, max = 19.21, var = 1.00 | |
| 0.54.385.636 I 00948: f32: Qcur_normed-23 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.90, avg = -0.01, max = 18.61, var = 0.94 | |
| 0.54.446.935 I 00949: f32: Qcur_pos-23 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.90, avg = -0.01, max = 18.61, var = 0.94 | |
| 0.54.464.130 I 00950: f32: Vcur-23 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -422.94, avg = -0.05, max = 359.31, var = 9.90 | |
| 0.54.471.679 I 00951: f32: Vcur-23 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -422.94, avg = -0.05, max = 359.31, var = 9.90 | |
| 0.54.479.699 I 00952: f32: norm-23 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.02, max = 22.17, var = 1.00 | |
| 0.54.487.639 I 00953: f32: Kcur_normed-23 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.43, var = 0.00 | |
| 0.54.495.606 I 00954: f32: Kcur_pos-23 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.43, var = 0.00 | |
| 0.54.503.150 I 00955: f32: Vcur-23 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -422.94, avg = -0.05, max = 359.31, var = 9.90 | |
| 0.54.511.029 I 00956: f32: Vcur_normed-23 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.02, max = 22.17, var = 1.00 | |
| 0.54.518.602 I 00957: f32: Kcur_pos-23 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.43, var = 0.00 | |
| 0.54.576.851 I 00958: f16: cache_k_l23 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.43, var = 0.00 | |
| 0.54.584.167 I 00959: f32: Vcur_normed-23 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.02, max = 22.17, var = 1.00 | |
| 0.54.642.304 I 00960: f16: cache_v_l23 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.00, max = 22.17, var = 0.12 | |
| 0.54.702.441 I 00961: f32: Qcur_pos-23 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.90, avg = -0.01, max = 18.61, var = 0.94 | |
| 0.54.762.828 I 00962: f32: Qcur_pos-23 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.90, avg = -0.01, max = 18.61, var = 0.94 | |
| 0.54.770.162 I 00963: f16: cache_k_l23 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.43, var = 0.00 | |
| 0.54.777.496 I 00964: f16: cache_k_l23 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.43, var = 0.00 | |
| 0.54.784.772 I 00965: f16: cache_v_l23 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.02, max = 22.17, var = 1.00 | |
| 0.54.792.059 I 00966: f16: cache_v_l23 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.02, max = 22.17, var = 1.00 | |
| 0.54.871.178 I 00967: f32: __fattn__-23 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.01, max = 22.00, var = 0.43 | |
| 0.54.931.648 I 00968: f32: kqv_out-23 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.25, avg = -0.01, max = 22.00, var = 0.43 | |
| 0.55.024.983 I 00969: f32: node_968 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.24, avg = 0.02, max = 100.56, var = 3.48 | |
| 0.55.045.360 I 00970: f32: norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.68, avg = 0.01, max = 52.47, var = 1.00 | |
| 0.55.065.108 I 00971: f32: attn_post_norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.90, avg = 0.02, max = 95.10, var = 1.32 | |
| 0.55.085.395 I 00972: f32: attn_out-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -178.20, avg = 0.08, max = 511.65, var = 43.14 | |
| 0.55.105.437 I 00973: f32: norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.54, avg = 0.01, max = 68.02, var = 1.00 | |
| 0.55.125.497 I 00974: f32: ffn_norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.80, avg = 0.00, max = 21.06, var = 0.09 | |
| 0.55.300.862 I 00975: f32: ffn_gate-23 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.03, avg = -0.39, max = 14.87, var = 0.42 | |
| 0.55.476.164 I 00976: f32: ffn_up-23 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.30, avg = 0.00, max = 23.18, var = 0.22 | |
| 0.55.556.981 I 00977: f32: ffn_geglu-23 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -554.70, avg = 0.00, max = 30.71, var = 0.04 | |
| 0.55.673.386 I 00978: f32: ffn_out-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -76.56, avg = -0.00, max = 88.13, var = 0.23 | |
| 0.55.693.760 I 00979: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.31, avg = -0.01, max = 46.77, var = 1.00 | |
| 0.55.713.847 I 00980: f32: ffn_post_norm-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.91, avg = 0.01, max = 55.17, var = 0.67 | |
| 0.55.734.088 I 00981: f32: node_980 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -187.88, avg = 0.08, max = 558.98, var = 51.13 | |
| 0.55.754.751 I 00982: f32: l_out-23 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -127.70, avg = 0.06, max = 379.93, var = 23.62 | |
| 0.55.775.155 I 00983: f32: norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.03, avg = 0.01, max = 68.36, var = 1.00 | |
| 0.55.795.225 I 00984: f32: attn_norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -103.81, avg = -0.00, max = 148.36, var = 3.09 | |
| 0.55.862.531 I 00985: f32: Qcur-24 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -178.32, avg = -0.04, max = 131.61, var = 15.68 | |
| 0.55.892.744 I 00986: f32: Qcur-24 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -178.32, avg = -0.04, max = 131.61, var = 15.68 | |
| 0.55.923.863 I 00987: f32: norm-24 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.70, avg = -0.01, max = 13.47, var = 1.00 | |
| 0.55.954.349 I 00988: f32: Qcur_normed-24 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.24, avg = -0.01, max = 14.00, var = 1.08 | |
| 0.55.985.081 I 00989: f32: Qcur_pos-24 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.43, avg = 0.01, max = 14.08, var = 1.08 | |
| 0.56.019.043 I 00990: f32: Kcur-24 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -417.30, avg = -0.04, max = 354.00, var = 23.22 | |
| 0.56.034.148 I 00991: f32: Kcur-24 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -417.30, avg = -0.04, max = 354.00, var = 23.22 | |
| 0.56.049.823 I 00992: f32: norm-24 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.59, avg = -0.01, max = 14.60, var = 1.00 | |
| 0.56.065.280 I 00993: f32: Kcur_normed-24 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = -0.00, max = 1.75, var = 0.01 | |
| 0.56.080.837 I 00994: f32: Kcur_pos-24 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.75, var = 0.01 | |
| 0.56.114.824 I 00995: f32: Vcur-24 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -448.23, avg = 0.02, max = 530.53, var = 10.46 | |
| 0.56.129.933 I 00996: f32: Vcur-24 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -448.23, avg = 0.02, max = 530.53, var = 10.46 | |
| 0.56.145.708 I 00997: f32: Vcur_normed-24 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.95, var = 1.00 | |
| 0.56.160.844 I 00998: f32: Kcur_pos-24 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.75, var = 0.01 | |
| 0.56.204.850 I 00999: f16: cache_k_l24 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.75, var = 0.00 | |
| 0.56.219.837 I 01000: f32: Vcur_normed-24 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.95, var = 1.00 | |
| 0.56.263.630 I 01001: f16: cache_v_l24 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.00, max = 15.95, var = 0.33 | |
| 0.56.293.525 I 01002: f32: Qcur_pos-24 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.43, avg = 0.01, max = 14.08, var = 1.08 | |
| 0.56.323.724 I 01003: f32: Qcur_pos-24 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.43, avg = 0.01, max = 14.08, var = 1.08 | |
| 0.56.338.388 I 01004: f16: cache_k_l24 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.75, var = 0.01 | |
| 0.56.353.001 I 01005: f16: cache_k_l24 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.75, var = 0.01 | |
| 0.56.367.555 I 01006: f16: cache_v_l24 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.95, var = 1.00 | |
| 0.56.382.120 I 01007: f16: cache_v_l24 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.01, max = 15.95, var = 1.00 | |
| 0.56.422.624 I 01008: f32: __fattn__-24 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.64, avg = 0.01, max = 15.80, var = 0.32 | |
| 0.56.452.850 I 01009: f32: kqv_out-24 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.64, avg = 0.01, max = 15.80, var = 0.32 | |
| 0.56.509.495 I 01010: f32: node_1009 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.18, avg = 0.00, max = 45.24, var = 0.98 | |
| 0.56.529.874 I 01011: f32: norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.25, avg = 0.00, max = 44.06, var = 1.00 | |
| 0.56.549.989 I 01012: f32: attn_post_norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.96, avg = 0.00, max = 36.84, var = 0.25 | |
| 0.56.570.229 I 01013: f32: attn_out-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -132.05, avg = 0.06, max = 408.91, var = 27.12 | |
| 0.56.590.280 I 01014: f32: norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.68, avg = 0.01, max = 68.74, var = 1.00 | |
| 0.56.610.340 I 01015: f32: ffn_norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.22, avg = 0.00, max = 17.15, var = 0.12 | |
| 0.56.785.723 I 01016: f32: ffn_gate-24 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.55, avg = -0.60, max = 13.36, var = 0.72 | |
| 0.56.961.445 I 01017: f32: ffn_up-24 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.59, avg = 0.00, max = 28.51, var = 0.30 | |
| 0.57.042.420 I 01018: f32: ffn_geglu-24 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -117.02, avg = 0.00, max = 251.84, var = 0.02 | |
| 0.57.158.710 I 01019: f32: ffn_out-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -83.27, avg = 0.00, max = 100.46, var = 0.15 | |
| 0.57.179.087 I 01020: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.88, avg = 0.01, max = 49.33, var = 1.00 | |
| 0.57.199.198 I 01021: f32: ffn_post_norm-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.43, avg = 0.01, max = 55.88, var = 0.65 | |
| 0.57.219.439 I 01022: f32: node_1021 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -135.37, avg = 0.07, max = 451.63, var = 33.24 | |
| 0.57.239.850 I 01023: f32: l_out-24 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -110.52, avg = 0.06, max = 368.71, var = 22.16 | |
| 0.57.260.634 I 01024: f32: norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.13, avg = 0.01, max = 69.26, var = 1.00 | |
| 0.57.280.725 I 01025: f32: attn_norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -175.59, avg = -0.00, max = 135.69, var = 3.40 | |
| 0.57.351.183 I 01026: f32: Qcur-25 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -182.11, avg = 0.04, max = 143.39, var = 18.66 | |
| 0.57.381.373 I 01027: f32: Qcur-25 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -182.11, avg = 0.04, max = 143.39, var = 18.66 | |
| 0.57.411.938 I 01028: f32: norm-25 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.71, avg = 0.01, max = 13.51, var = 1.00 | |
| 0.57.441.534 I 01029: f32: Qcur_normed-25 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.04, avg = 0.01, max = 13.82, var = 1.05 | |
| 0.57.471.527 I 01030: f32: Qcur_pos-25 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.30, avg = 0.01, max = 13.83, var = 1.05 | |
| 0.57.505.897 I 01031: f32: Kcur-25 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -246.86, avg = -0.03, max = 313.74, var = 27.49 | |
| 0.57.521.002 I 01032: f32: Kcur-25 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -246.86, avg = -0.03, max = 313.74, var = 27.49 | |
| 0.57.536.466 I 01033: f32: norm-25 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.09, avg = -0.01, max = 13.17, var = 1.00 | |
| 0.57.551.454 I 01034: f32: Kcur_normed-25 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.24, avg = -0.00, max = 1.61, var = 0.02 | |
| 0.57.566.598 I 01035: f32: Kcur_pos-25 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.22, avg = -0.00, max = 1.61, var = 0.02 | |
| 0.57.600.841 I 01036: f32: Vcur-25 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -576.45, avg = -0.04, max = 604.07, var = 13.91 | |
| 0.57.615.936 I 01037: f32: Vcur-25 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -576.45, avg = -0.04, max = 604.07, var = 13.91 | |
| 0.57.631.475 I 01038: f32: Vcur_normed-25 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = -0.01, max = 15.97, var = 1.00 | |
| 0.57.646.608 I 01039: f32: Kcur_pos-25 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.22, avg = -0.00, max = 1.61, var = 0.02 | |
| 0.57.690.640 I 01040: f16: cache_k_l25 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.22, avg = -0.00, max = 1.61, var = 0.01 | |
| 0.57.705.287 I 01041: f32: Vcur_normed-25 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = -0.01, max = 15.97, var = 1.00 | |
| 0.57.749.908 I 01042: f16: cache_v_l25 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = -0.00, max = 15.98, var = 0.33 | |
| 0.57.779.190 I 01043: f32: Qcur_pos-25 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.30, avg = 0.01, max = 13.83, var = 1.05 | |
| 0.57.808.492 I 01044: f32: Qcur_pos-25 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.30, avg = 0.01, max = 13.83, var = 1.05 | |
| 0.57.823.090 I 01045: f16: cache_k_l25 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.22, avg = -0.00, max = 1.61, var = 0.02 | |
| 0.57.837.699 I 01046: f16: cache_k_l25 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.22, avg = -0.00, max = 1.61, var = 0.02 | |
| 0.57.852.236 I 01047: f16: cache_v_l25 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = -0.01, max = 15.98, var = 1.00 | |
| 0.57.866.776 I 01048: f16: cache_v_l25 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = -0.01, max = 15.98, var = 1.00 | |
| 0.57.907.196 I 01049: f32: __fattn__-25 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.34, avg = -0.01, max = 15.55, var = 0.31 | |
| 0.57.937.311 I 01050: f32: kqv_out-25 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.34, avg = -0.01, max = 15.55, var = 0.31 | |
| 0.57.994.142 I 01051: f32: node_1050 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.78, avg = 0.00, max = 50.36, var = 1.04 | |
| 0.58.014.063 I 01052: f32: norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.58, avg = 0.00, max = 44.75, var = 1.00 | |
| 0.58.033.634 I 01053: f32: attn_post_norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.54, avg = 0.00, max = 41.78, var = 0.32 | |
| 0.58.053.501 I 01054: f32: attn_out-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -111.66, avg = 0.06, max = 407.08, var = 26.07 | |
| 0.58.073.421 I 01055: f32: norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.93, avg = 0.01, max = 69.65, var = 1.00 | |
| 0.58.092.944 I 01056: f32: ffn_norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.08, avg = -0.00, max = 16.54, var = 0.10 | |
| 0.58.268.754 I 01057: f32: ffn_gate-25 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.99, avg = -0.46, max = 13.67, var = 0.55 | |
| 0.58.444.419 I 01058: f32: ffn_up-25 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.88, avg = -0.00, max = 28.53, var = 0.20 | |
| 0.58.525.720 I 01059: f32: ffn_geglu-25 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -471.31, avg = 0.00, max = 115.78, var = 0.03 | |
| 0.58.641.987 I 01060: f32: ffn_out-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -109.65, avg = 0.00, max = 105.99, var = 0.14 | |
| 0.58.661.872 I 01061: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.61, avg = 0.01, max = 48.80, var = 1.00 | |
| 0.58.681.553 I 01062: f32: ffn_post_norm-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.72, avg = 0.01, max = 64.82, var = 0.86 | |
| 0.58.701.279 I 01063: f32: node_1062 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -116.54, avg = 0.07, max = 462.73, var = 33.02 | |
| 0.58.721.235 I 01064: f32: l_out-25 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -90.59, avg = 0.05, max = 359.70, var = 19.95 | |
| 0.58.741.184 I 01065: f32: norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.96, avg = 0.01, max = 70.09, var = 1.00 | |
| 0.58.760.719 I 01066: f32: attn_norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -159.23, avg = -0.01, max = 192.34, var = 4.04 | |
| 0.58.828.426 I 01067: f32: Qcur-26 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -122.58, avg = -0.00, max = 148.87, var = 17.13 | |
| 0.58.858.623 I 01068: f32: Qcur-26 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -122.58, avg = -0.00, max = 148.87, var = 17.13 | |
| 0.58.888.972 I 01069: f32: norm-26 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.04, avg = -0.00, max = 14.23, var = 1.00 | |
| 0.58.918.373 I 01070: f32: Qcur_normed-26 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.14, avg = -0.00, max = 14.35, var = 1.02 | |
| 0.58.948.365 I 01071: f32: Qcur_pos-26 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.14, avg = 0.01, max = 14.29, var = 1.02 | |
| 0.58.982.337 I 01072: f32: Kcur-26 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -313.03, avg = -0.04, max = 293.79, var = 23.04 | |
| 0.58.997.444 I 01073: f32: Kcur-26 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -313.03, avg = -0.04, max = 293.79, var = 23.04 | |
| 0.59.012.890 I 01074: f32: norm-26 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.55, avg = -0.01, max = 11.74, var = 1.00 | |
| 0.59.027.873 I 01075: f32: Kcur_normed-26 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = -0.00, max = 1.46, var = 0.02 | |
| 0.59.043.027 I 01076: f32: Kcur_pos-26 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.42, var = 0.02 | |
| 0.59.077.091 I 01077: f32: Vcur-26 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -730.17, avg = -0.01, max = 706.60, var = 15.05 | |
| 0.59.092.195 I 01078: f32: Vcur-26 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -730.17, avg = -0.01, max = 706.60, var = 15.05 | |
| 0.59.107.578 I 01079: f32: Vcur_normed-26 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.59.122.279 I 01080: f32: Kcur_pos-26 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.42, var = 0.02 | |
| 0.59.166.321 I 01081: f16: cache_k_l26 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.42, var = 0.01 | |
| 0.59.180.977 I 01082: f32: Vcur_normed-26 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.59.224.885 I 01083: f16: cache_v_l26 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.98, var = 0.33 | |
| 0.59.254.157 I 01084: f32: Qcur_pos-26 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.14, avg = 0.01, max = 14.29, var = 1.02 | |
| 0.59.283.452 I 01085: f32: Qcur_pos-26 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.14, avg = 0.01, max = 14.29, var = 1.02 | |
| 0.59.298.070 I 01086: f16: cache_k_l26 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.42, var = 0.02 | |
| 0.59.312.716 I 01087: f16: cache_k_l26 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.32, avg = -0.00, max = 1.42, var = 0.02 | |
| 0.59.327.290 I 01088: f16: cache_v_l26 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.59.341.841 I 01089: f16: cache_v_l26 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.98, var = 1.00 | |
| 0.59.381.680 I 01090: f32: __fattn__-26 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.73, avg = -0.01, max = 15.62, var = 0.29 | |
| 0.59.411.014 I 01091: f32: kqv_out-26 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.73, avg = -0.01, max = 15.62, var = 0.29 | |
| 0.59.468.032 I 01092: f32: node_1091 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.52, avg = -0.00, max = 39.82, var = 0.64 | |
| 0.59.488.394 I 01093: f32: norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.20, avg = -0.00, max = 43.90, var = 1.00 | |
| 0.59.508.025 I 01094: f32: attn_post_norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.13, avg = 0.01, max = 59.67, var = 0.68 | |
| 0.59.527.793 I 01095: f32: attn_out-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.08, avg = 0.06, max = 402.51, var = 25.70 | |
| 0.59.547.698 I 01096: f32: norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.40, avg = 0.01, max = 70.12, var = 1.00 | |
| 0.59.567.279 I 01097: f32: ffn_norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.28, avg = -0.00, max = 20.15, var = 0.15 | |
| 0.59.742.950 I 01098: f32: ffn_gate-26 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.06, avg = -0.59, max = 13.13, var = 0.68 | |
| 0.59.918.497 I 01099: f32: ffn_up-26 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.66, avg = -0.00, max = 29.83, var = 0.26 | |
| 0.59.999.765 I 01100: f32: ffn_geglu-26 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -106.60, avg = 0.00, max = 197.90, var = 0.02 | |
| 1.00.116.146 I 01101: f32: ffn_out-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -144.66, avg = 0.00, max = 145.81, var = 0.52 | |
| 1.00.136.123 I 01102: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.06, avg = 0.00, max = 55.23, var = 1.00 | |
| 1.00.155.695 I 01103: f32: ffn_post_norm-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.61, avg = 0.01, max = 107.00, var = 1.94 | |
| 1.00.175.576 I 01104: f32: node_1103 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -104.40, avg = 0.07, max = 498.51, var = 38.36 | |
| 1.00.195.525 I 01105: f32: l_out-26 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -67.29, avg = 0.05, max = 321.30, var = 15.93 | |
| 1.00.215.506 I 01106: f32: norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.92, avg = 0.01, max = 70.62, var = 1.00 | |
| 1.00.235.005 I 01107: f32: attn_norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -225.35, avg = -0.02, max = 237.80, var = 4.06 | |
| 1.00.302.511 I 01108: f32: Qcur-27 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -197.38, avg = -0.06, max = 168.93, var = 17.50 | |
| 1.00.332.709 I 01109: f32: Qcur-27 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -197.38, avg = -0.06, max = 168.93, var = 17.50 | |
| 1.00.363.789 I 01110: f32: norm-27 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.07, avg = -0.02, max = 12.80, var = 1.00 | |
| 1.00.394.260 I 01111: f32: Qcur_normed-27 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.76, avg = -0.01, max = 12.50, var = 0.95 | |
| 1.00.425.082 I 01112: f32: Qcur_pos-27 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.91, avg = -0.00, max = 13.64, var = 0.95 | |
| 1.00.458.866 I 01113: f32: Kcur-27 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -375.42, avg = -0.02, max = 369.45, var = 24.04 | |
| 1.00.473.977 I 01114: f32: Kcur-27 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -375.42, avg = -0.02, max = 369.45, var = 24.04 | |
| 1.00.489.290 I 01115: f32: norm-27 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.37, avg = -0.01, max = 11.70, var = 1.00 | |
| 1.00.504.252 I 01116: f32: Kcur_normed-27 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.33, avg = -0.00, max = 1.50, var = 0.02 | |
| 1.00.519.418 I 01117: f32: Kcur_pos-27 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 1.53, var = 0.02 | |
| 1.00.553.685 I 01118: f32: Vcur-27 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -863.87, avg = 0.01, max = 793.09, var = 14.94 | |
| 1.00.568.793 I 01119: f32: Vcur-27 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -863.87, avg = 0.01, max = 793.09, var = 14.94 | |
| 1.00.584.447 I 01120: f32: Vcur_normed-27 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.00.599.583 I 01121: f32: Kcur_pos-27 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 1.53, var = 0.02 | |
| 1.00.643.608 I 01122: f16: cache_k_l27 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 1.53, var = 0.01 | |
| 1.00.658.266 I 01123: f32: Vcur_normed-27 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.00.702.180 I 01124: f16: cache_v_l27 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 0.33 | |
| 1.00.731.456 I 01125: f32: Qcur_pos-27 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.91, avg = -0.00, max = 13.64, var = 0.95 | |
| 1.00.760.756 I 01126: f32: Qcur_pos-27 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.91, avg = -0.00, max = 13.64, var = 0.95 | |
| 1.00.775.377 I 01127: f16: cache_k_l27 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 1.53, var = 0.02 | |
| 1.00.789.978 I 01128: f16: cache_k_l27 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.34, avg = -0.00, max = 1.53, var = 0.02 | |
| 1.00.804.527 I 01129: f16: cache_v_l27 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.00.819.074 I 01130: f16: cache_v_l27 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.00.859.053 I 01131: f32: __fattn__-27 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.70, avg = -0.00, max = 15.70, var = 0.26 | |
| 1.00.888.441 I 01132: f32: kqv_out-27 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.70, avg = -0.00, max = 15.70, var = 0.26 | |
| 1.00.945.623 I 01133: f32: node_1132 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.09, avg = 0.01, max = 53.91, var = 0.78 | |
| 1.00.965.547 I 01134: f32: norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.59, avg = 0.01, max = 49.41, var = 1.00 | |
| 1.00.985.096 I 01135: f32: attn_post_norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.03, avg = 0.01, max = 65.62, var = 0.62 | |
| 1.01.004.810 I 01136: f32: attn_out-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -72.35, avg = 0.06, max = 377.59, var = 21.25 | |
| 1.01.024.511 I 01137: f32: norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.72, avg = 0.01, max = 70.91, var = 1.00 | |
| 1.01.044.111 I 01138: f32: ffn_norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.86, avg = -0.00, max = 19.68, var = 0.08 | |
| 1.01.219.494 I 01139: f32: ffn_gate-27 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.98, avg = -0.34, max = 15.37, var = 0.37 | |
| 1.01.394.573 I 01140: f32: ffn_up-27 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -42.84, avg = 0.00, max = 52.50, var = 0.15 | |
| 1.01.475.818 I 01141: f32: ffn_geglu-27 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -326.79, avg = -0.00, max = 671.62, var = 0.06 | |
| 1.01.592.433 I 01142: f32: ffn_out-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -133.92, avg = 0.00, max = 166.26, var = 0.28 | |
| 1.01.612.314 I 01143: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.08, avg = 0.01, max = 52.13, var = 1.00 | |
| 1.01.631.889 I 01144: f32: ffn_post_norm-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.90, avg = 0.02, max = 112.40, var = 2.21 | |
| 1.01.651.583 I 01145: f32: node_1144 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -94.06, avg = 0.08, max = 478.80, var = 33.61 | |
| 1.01.671.598 I 01146: f32: l_out-27 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.42, avg = 0.05, max = 297.38, var = 12.97 | |
| 1.01.691.546 I 01147: f32: norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.54, avg = 0.01, max = 70.94, var = 1.00 | |
| 1.01.711.061 I 01148: f32: attn_norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.42, avg = -0.01, max = 155.90, var = 2.00 | |
| 1.01.778.652 I 01149: f32: Qcur-28 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -108.36, avg = 0.02, max = 117.47, var = 8.46 | |
| 1.01.808.849 I 01150: f32: Qcur-28 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -108.36, avg = 0.02, max = 117.47, var = 8.46 | |
| 1.01.839.193 I 01151: f32: norm-28 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.00, avg = 0.01, max = 14.29, var = 1.00 | |
| 1.01.868.784 I 01152: f32: Qcur_normed-28 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.34, avg = 0.01, max = 13.62, var = 0.91 | |
| 1.01.898.746 I 01153: f32: Qcur_pos-28 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.34, avg = 0.00, max = 13.63, var = 0.91 | |
| 1.01.932.874 I 01154: f32: Kcur-28 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -241.89, avg = -0.05, max = 313.69, var = 10.10 | |
| 1.01.947.980 I 01155: f32: Kcur-28 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -241.89, avg = -0.05, max = 313.69, var = 10.10 | |
| 1.01.963.292 I 01156: f32: norm-28 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.02, avg = -0.02, max = 11.96, var = 1.00 | |
| 1.01.978.396 I 01157: f32: Kcur_normed-28 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.18, avg = -0.00, max = 1.56, var = 0.02 | |
| 1.01.993.627 I 01158: f32: Kcur_pos-28 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.17, avg = -0.00, max = 1.56, var = 0.02 | |
| 1.02.027.755 I 01159: f32: Vcur-28 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -656.89, avg = 0.01, max = 622.44, var = 5.89 | |
| 1.02.042.862 I 01160: f32: Vcur-28 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -656.89, avg = 0.01, max = 622.44, var = 5.89 | |
| 1.02.058.188 I 01161: f32: Vcur_normed-28 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.02.072.866 I 01162: f32: Kcur_pos-28 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.17, avg = -0.00, max = 1.56, var = 0.02 | |
| 1.02.116.884 I 01163: f16: cache_k_l28 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.17, avg = -0.00, max = 1.56, var = 0.01 | |
| 1.02.131.541 I 01164: f32: Vcur_normed-28 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.02.175.454 I 01165: f16: cache_v_l28 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 0.33 | |
| 1.02.204.743 I 01166: f32: Qcur_pos-28 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.34, avg = 0.00, max = 13.63, var = 0.91 | |
| 1.02.234.008 I 01167: f32: Qcur_pos-28 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.34, avg = 0.00, max = 13.63, var = 0.91 | |
| 1.02.248.627 I 01168: f16: cache_k_l28 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.17, avg = -0.00, max = 1.56, var = 0.02 | |
| 1.02.263.215 I 01169: f16: cache_k_l28 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.17, avg = -0.00, max = 1.56, var = 0.02 | |
| 1.02.277.770 I 01170: f16: cache_v_l28 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.02.292.327 I 01171: f16: cache_v_l28 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.02.332.270 I 01172: f32: __fattn__-28 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.00, max = 15.95, var = 0.24 | |
| 1.02.361.572 I 01173: f32: kqv_out-28 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.00, max = 15.95, var = 0.24 | |
| 1.02.418.929 I 01174: f32: node_1173 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.30, avg = 0.01, max = 74.90, var = 0.72 | |
| 1.02.439.007 I 01175: f32: norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.18, avg = 0.01, max = 56.77, var = 1.00 | |
| 1.02.458.544 I 01176: f32: attn_post_norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.01, avg = 0.03, max = 143.71, var = 4.55 | |
| 1.02.478.231 I 01177: f32: attn_out-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.47, avg = 0.08, max = 437.86, var = 29.44 | |
| 1.02.498.146 I 01178: f32: norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.15, avg = 0.02, max = 69.90, var = 1.00 | |
| 1.02.517.688 I 01179: f32: ffn_norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.53, avg = -0.00, max = 24.28, var = 0.05 | |
| 1.02.694.432 I 01180: f32: ffn_gate-28 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.12, avg = -0.13, max = 18.97, var = 0.12 | |
| 1.02.874.249 I 01181: f32: ffn_up-28 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.85, avg = -0.00, max = 34.38, var = 0.09 | |
| 1.02.955.423 I 01182: f32: ffn_geglu-28 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.97, avg = -0.00, max = 319.03, var = 0.01 | |
| 1.03.071.908 I 01183: f32: ffn_out-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -128.48, avg = 0.00, max = 51.61, var = 0.06 | |
| 1.03.091.767 I 01184: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.35, avg = 0.01, max = 54.32, var = 1.00 | |
| 1.03.111.349 I 01185: f32: ffn_post_norm-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -122.56, avg = 0.04, max = 178.23, var = 7.33 | |
| 1.03.131.044 I 01186: f32: node_1185 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -150.70, avg = 0.12, max = 597.52, var = 61.57 | |
| 1.03.151.034 I 01187: f32: l_out-28 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -65.64, avg = 0.05, max = 260.25, var = 11.68 | |
| 1.03.170.985 I 01188: f32: norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.12, avg = 0.02, max = 68.94, var = 1.00 | |
| 1.03.190.502 I 01189: f32: attn_norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -445.67, avg = -0.01, max = 282.68, var = 2.85 | |
| 1.03.325.273 I 01190: f32: Qcur-29 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -398.92, avg = -0.01, max = 545.73, var = 7.48 | |
| 1.03.385.647 I 01191: f32: Qcur-29 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -398.92, avg = -0.01, max = 545.73, var = 7.48 | |
| 1.03.446.466 I 01192: f32: norm-29 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.52, avg = -0.00, max = 21.47, var = 1.00 | |
| 1.03.505.803 I 01193: f32: Qcur_normed-29 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.52, avg = -0.00, max = 21.47, var = 1.00 | |
| 1.03.565.278 I 01194: f32: Qcur_pos-29 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.52, avg = -0.00, max = 21.47, var = 1.00 | |
| 1.03.582.718 I 01195: f32: Vcur-29 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -148.44, avg = -0.03, max = 582.75, var = 6.81 | |
| 1.03.590.266 I 01196: f32: Vcur-29 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -148.44, avg = -0.03, max = 582.75, var = 6.81 | |
| 1.03.598.094 I 01197: f32: norm-29 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.80, avg = -0.01, max = 22.23, var = 1.00 | |
| 1.03.605.786 I 01198: f32: Kcur_normed-29 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.39, var = 0.00 | |
| 1.03.613.372 I 01199: f32: Kcur_pos-29 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.39, var = 0.00 | |
| 1.03.620.711 I 01200: f32: Vcur-29 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -148.44, avg = -0.03, max = 582.75, var = 6.81 | |
| 1.03.628.423 I 01201: f32: Vcur_normed-29 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.80, avg = -0.01, max = 22.23, var = 1.00 | |
| 1.03.635.822 I 01202: f32: Kcur_pos-29 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.39, var = 0.00 | |
| 1.03.694.184 I 01203: f16: cache_k_l29 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.39, var = 0.00 | |
| 1.03.701.514 I 01204: f32: Vcur_normed-29 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.80, avg = -0.01, max = 22.23, var = 1.00 | |
| 1.03.759.874 I 01205: f16: cache_v_l29 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.80, avg = -0.00, max = 22.23, var = 0.12 | |
| 1.03.818.393 I 01206: f32: Qcur_pos-29 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.52, avg = -0.00, max = 21.47, var = 1.00 | |
| 1.03.877.149 I 01207: f32: Qcur_pos-29 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.52, avg = -0.00, max = 21.47, var = 1.00 | |
| 1.03.884.494 I 01208: f16: cache_k_l29 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.39, var = 0.00 | |
| 1.03.891.840 I 01209: f16: cache_k_l29 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.30, avg = -0.00, max = 1.39, var = 0.00 | |
| 1.03.899.130 I 01210: f16: cache_v_l29 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.80, avg = -0.01, max = 22.23, var = 1.00 | |
| 1.03.906.440 I 01211: f16: cache_v_l29 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.80, avg = -0.01, max = 22.23, var = 1.00 | |
| 1.03.985.645 I 01212: f32: __fattn__-29 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.47, avg = 0.01, max = 22.23, var = 0.30 | |
| 1.04.046.057 I 01213: f32: kqv_out-29 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.47, avg = 0.01, max = 22.23, var = 0.30 | |
| 1.04.139.861 I 01214: f32: node_1213 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.05, avg = -0.00, max = 42.11, var = 0.63 | |
| 1.04.159.932 I 01215: f32: norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.82, avg = -0.00, max = 49.98, var = 1.00 | |
| 1.04.179.619 I 01216: f32: attn_post_norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.14, avg = 0.01, max = 92.14, var = 1.84 | |
| 1.04.199.134 I 01217: f32: attn_out-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -118.78, avg = 0.07, max = 340.37, var = 20.98 | |
| 1.04.219.083 I 01218: f32: norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -46.53, avg = 0.01, max = 67.62, var = 1.00 | |
| 1.04.238.629 I 01219: f32: ffn_norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.81, avg = -0.00, max = 37.48, var = 0.07 | |
| 1.04.414.675 I 01220: f32: ffn_gate-29 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.18, avg = -0.15, max = 25.47, var = 0.15 | |
| 1.04.590.285 I 01221: f32: ffn_up-29 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.97, avg = 0.00, max = 48.54, var = 0.11 | |
| 1.04.671.635 I 01222: f32: ffn_geglu-29 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -285.25, avg = 0.00, max = 1235.96, var = 0.17 | |
| 1.04.788.047 I 01223: f32: ffn_out-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -145.08, avg = 0.00, max = 279.51, var = 0.48 | |
| 1.04.808.004 I 01224: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.51, avg = 0.01, max = 53.71, var = 1.00 | |
| 1.04.827.632 I 01225: f32: ffn_post_norm-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.59, avg = 0.02, max = 127.56, var = 2.66 | |
| 1.04.847.326 I 01226: f32: node_1225 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -154.92, avg = 0.09, max = 447.03, var = 35.39 | |
| 1.04.867.326 I 01227: f32: l_out-29 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -85.93, avg = 0.05, max = 247.96, var = 10.89 | |
| 1.04.887.347 I 01228: f32: norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.01, avg = 0.01, max = 67.19, var = 1.00 | |
| 1.04.906.687 I 01229: f32: attn_norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -702.30, avg = -0.02, max = 479.30, var = 2.39 | |
| 1.04.974.073 I 01230: f32: Qcur-30 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -127.82, avg = -0.01, max = 82.85, var = 6.41 | |
| 1.05.004.267 I 01231: f32: Qcur-30 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -127.82, avg = -0.01, max = 82.85, var = 6.41 | |
| 1.05.034.688 I 01232: f32: norm-30 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.69, avg = -0.00, max = 13.19, var = 1.00 | |
| 1.05.064.281 I 01233: f32: Qcur_normed-30 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.15, avg = -0.00, max = 13.60, var = 1.06 | |
| 1.05.094.241 I 01234: f32: Qcur_pos-30 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.05, avg = -0.01, max = 14.57, var = 1.06 | |
| 1.05.128.172 I 01235: f32: Kcur-30 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -226.52, avg = 0.02, max = 223.21, var = 9.03 | |
| 1.05.143.280 I 01236: f32: Kcur-30 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -226.52, avg = 0.02, max = 223.21, var = 9.03 | |
| 1.05.158.902 I 01237: f32: norm-30 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.79, avg = 0.01, max = 12.01, var = 1.00 | |
| 1.05.173.912 I 01238: f32: Kcur_normed-30 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = 0.00, max = 1.46, var = 0.01 | |
| 1.05.189.062 I 01239: f32: Kcur_pos-30 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.44, avg = 0.00, max = 1.46, var = 0.01 | |
| 1.05.223.092 I 01240: f32: Vcur-30 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -645.62, avg = -0.00, max = 739.01, var = 7.63 | |
| 1.05.238.200 I 01241: f32: Vcur-30 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -645.62, avg = -0.00, max = 739.01, var = 7.63 | |
| 1.05.253.537 I 01242: f32: Vcur_normed-30 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = -0.00, max = 15.94, var = 1.00 | |
| 1.05.268.224 I 01243: f32: Kcur_pos-30 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.44, avg = 0.00, max = 1.46, var = 0.01 | |
| 1.05.312.289 I 01244: f16: cache_k_l30 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.44, avg = 0.00, max = 1.46, var = 0.00 | |
| 1.05.326.948 I 01245: f32: Vcur_normed-30 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = -0.00, max = 15.94, var = 1.00 | |
| 1.05.370.872 I 01246: f16: cache_v_l30 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = -0.00, max = 15.95, var = 0.33 | |
| 1.05.400.162 I 01247: f32: Qcur_pos-30 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.05, avg = -0.01, max = 14.57, var = 1.06 | |
| 1.05.429.446 I 01248: f32: Qcur_pos-30 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.05, avg = -0.01, max = 14.57, var = 1.06 | |
| 1.05.444.060 I 01249: f16: cache_k_l30 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.44, avg = 0.00, max = 1.46, var = 0.01 | |
| 1.05.458.665 I 01250: f16: cache_k_l30 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.44, avg = 0.00, max = 1.46, var = 0.01 | |
| 1.05.473.253 I 01251: f16: cache_v_l30 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = -0.00, max = 15.95, var = 1.00 | |
| 1.05.487.804 I 01252: f16: cache_v_l30 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.87, avg = -0.00, max = 15.95, var = 1.00 | |
| 1.05.528.058 I 01253: f32: __fattn__-30 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.85, avg = -0.00, max = 15.95, var = 0.35 | |
| 1.05.557.359 I 01254: f32: kqv_out-30 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.85, avg = -0.00, max = 15.95, var = 0.35 | |
| 1.05.614.970 I 01255: f32: node_1254 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.79, avg = 0.00, max = 60.53, var = 1.26 | |
| 1.05.634.837 I 01256: f32: norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.14, avg = 0.00, max = 54.80, var = 1.00 | |
| 1.05.654.415 I 01257: f32: attn_post_norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.21, avg = 0.01, max = 79.21, var = 0.97 | |
| 1.05.674.128 I 01258: f32: attn_out-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.58, avg = 0.06, max = 317.74, var = 17.46 | |
| 1.05.694.050 I 01259: f32: norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.14, avg = 0.01, max = 67.00, var = 1.00 | |
| 1.05.713.554 I 01260: f32: ffn_norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -78.35, avg = -0.00, max = 20.29, var = 0.10 | |
| 1.05.889.436 I 01261: f32: ffn_gate-30 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.85, avg = -0.36, max = 37.52, var = 0.32 | |
| 1.06.065.095 I 01262: f32: ffn_up-30 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.37, avg = -0.00, max = 34.43, var = 0.21 | |
| 1.06.146.182 I 01263: f32: ffn_geglu-30 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -587.27, avg = 0.00, max = 134.00, var = 0.06 | |
| 1.06.262.892 I 01264: f32: ffn_out-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -108.44, avg = 0.01, max = 109.23, var = 0.21 | |
| 1.06.282.788 I 01265: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.77, avg = 0.02, max = 46.82, var = 1.00 | |
| 1.06.302.471 I 01266: f32: ffn_post_norm-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.53, avg = 0.01, max = 69.14, var = 0.84 | |
| 1.06.322.175 I 01267: f32: node_1266 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -136.11, avg = 0.07, max = 377.99, var = 23.55 | |
| 1.06.342.194 I 01268: f32: l_out-30 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -94.64, avg = 0.05, max = 262.82, var = 11.39 | |
| 1.06.362.143 I 01269: f32: norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.55, avg = 0.01, max = 66.23, var = 1.00 | |
| 1.06.381.652 I 01270: f32: attn_norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -679.88, avg = -0.01, max = 156.51, var = 1.67 | |
| 1.06.449.107 I 01271: f32: Qcur-31 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.54, avg = -0.00, max = 68.84, var = 4.17 | |
| 1.06.479.312 I 01272: f32: Qcur-31 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.54, avg = -0.00, max = 68.84, var = 4.17 | |
| 1.06.509.659 I 01273: f32: norm-31 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.09, avg = -0.00, max = 13.90, var = 1.00 | |
| 1.06.539.129 I 01274: f32: Qcur_normed-31 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.53, avg = -0.00, max = 14.34, var = 1.06 | |
| 1.06.568.780 I 01275: f32: Qcur_pos-31 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.74, avg = 0.01, max = 14.53, var = 1.06 | |
| 1.06.602.939 I 01276: f32: Kcur-31 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -206.02, avg = 0.03, max = 177.13, var = 5.16 | |
| 1.06.617.700 I 01277: f32: Kcur-31 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -206.02, avg = 0.03, max = 177.13, var = 5.16 | |
| 1.06.633.008 I 01278: f32: norm-31 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.95, avg = 0.01, max = 12.31, var = 1.00 | |
| 1.06.647.971 I 01279: f32: Kcur_normed-31 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = 0.00, max = 1.49, var = 0.01 | |
| 1.06.663.134 I 01280: f32: Kcur_pos-31 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = 0.00, max = 1.49, var = 0.01 | |
| 1.06.697.121 I 01281: f32: Vcur-31 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -492.62, avg = -0.01, max = 507.24, var = 5.26 | |
| 1.06.712.220 I 01282: f32: Vcur-31 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -492.62, avg = -0.01, max = 507.24, var = 5.26 | |
| 1.06.727.570 I 01283: f32: Vcur_normed-31 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 1.00 | |
| 1.06.742.268 I 01284: f32: Kcur_pos-31 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = 0.00, max = 1.49, var = 0.01 | |
| 1.06.786.347 I 01285: f16: cache_k_l31 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = 0.00, max = 1.49, var = 0.00 | |
| 1.06.800.997 I 01286: f32: Vcur_normed-31 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.97, var = 1.00 | |
| 1.06.844.967 I 01287: f16: cache_v_l31 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.98, var = 0.33 | |
| 1.06.874.249 I 01288: f32: Qcur_pos-31 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.74, avg = 0.01, max = 14.53, var = 1.06 | |
| 1.06.903.546 I 01289: f32: Qcur_pos-31 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.74, avg = 0.01, max = 14.53, var = 1.06 | |
| 1.06.918.162 I 01290: f16: cache_k_l31 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = 0.00, max = 1.49, var = 0.01 | |
| 1.06.932.787 I 01291: f16: cache_k_l31 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = 0.00, max = 1.49, var = 0.01 | |
| 1.06.947.358 I 01292: f16: cache_v_l31 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.98, var = 1.00 | |
| 1.06.961.904 I 01293: f16: cache_v_l31 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.98, var = 1.00 | |
| 1.07.001.866 I 01294: f32: __fattn__-31 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.71, avg = 0.00, max = 15.98, var = 0.27 | |
| 1.07.031.170 I 01295: f32: kqv_out-31 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.71, avg = 0.00, max = 15.98, var = 0.27 | |
| 1.07.088.575 I 01296: f32: node_1295 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.68, avg = 0.00, max = 76.66, var = 0.93 | |
| 1.07.108.492 I 01297: f32: norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.98, avg = 0.00, max = 58.15, var = 1.00 | |
| 1.07.128.106 I 01298: f32: attn_post_norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.54, avg = 0.01, max = 61.33, var = 0.61 | |
| 1.07.147.832 I 01299: f32: attn_out-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -98.52, avg = 0.06, max = 316.26, var = 16.61 | |
| 1.07.167.767 I 01300: f32: norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.82, avg = 0.01, max = 66.24, var = 1.00 | |
| 1.07.187.267 I 01301: f32: ffn_norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.61, avg = -0.00, max = 23.98, var = 0.17 | |
| 1.07.362.815 I 01302: f32: ffn_gate-31 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.83, avg = -0.58, max = 24.06, var = 0.53 | |
| 1.07.537.400 I 01303: f32: ffn_up-31 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -47.45, avg = -0.00, max = 37.57, var = 0.31 | |
| 1.07.618.536 I 01304: f32: ffn_geglu-31 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1024.92, avg = -0.00, max = 265.10, var = 0.12 | |
| 1.07.735.223 I 01305: f32: ffn_out-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -109.68, avg = 0.01, max = 119.14, var = 0.35 | |
| 1.07.754.963 I 01306: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.32, avg = 0.02, max = 47.62, var = 1.00 | |
| 1.07.774.586 I 01307: f32: ffn_post_norm-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.52, avg = 0.01, max = 64.36, var = 0.90 | |
| 1.07.794.282 I 01308: f32: node_1307 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -130.03, avg = 0.07, max = 372.86, var = 23.56 | |
| 1.07.814.271 I 01309: f32: l_out-31 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -98.54, avg = 0.05, max = 282.56, var = 13.53 | |
| 1.07.834.203 I 01310: f32: norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.11, avg = 0.01, max = 65.50, var = 1.00 | |
| 1.07.853.713 I 01311: f32: attn_norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -876.89, avg = -0.02, max = 199.77, var = 2.25 | |
| 1.07.921.771 I 01312: f32: Qcur-32 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.35, avg = -0.01, max = 105.26, var = 4.71 | |
| 1.07.951.967 I 01313: f32: Qcur-32 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -113.35, avg = -0.01, max = 105.26, var = 4.71 | |
| 1.07.982.454 I 01314: f32: norm-32 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.25, avg = -0.00, max = 14.75, var = 1.00 | |
| 1.08.012.062 I 01315: f32: Qcur_normed-32 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.36, avg = -0.00, max = 14.87, var = 1.02 | |
| 1.08.042.003 I 01316: f32: Qcur_pos-32 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.66, avg = 0.01, max = 14.88, var = 1.02 | |
| 1.08.076.227 I 01317: f32: Kcur-32 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -225.25, avg = -0.02, max = 217.10, var = 6.99 | |
| 1.08.091.336 I 01318: f32: Kcur-32 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -225.25, avg = -0.02, max = 217.10, var = 6.99 | |
| 1.08.106.714 I 01319: f32: norm-32 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.75, avg = -0.01, max = 11.90, var = 1.00 | |
| 1.08.121.685 I 01320: f32: Kcur_normed-32 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.48, var = 0.02 | |
| 1.08.136.839 I 01321: f32: Kcur_pos-32 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.48, var = 0.02 | |
| 1.08.170.994 I 01322: f32: Vcur-32 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -635.11, avg = -0.01, max = 691.79, var = 6.39 | |
| 1.08.186.115 I 01323: f32: Vcur-32 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -635.11, avg = -0.01, max = 691.79, var = 6.39 | |
| 1.08.201.475 I 01324: f32: Vcur_normed-32 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.01, max = 15.96, var = 1.00 | |
| 1.08.216.162 I 01325: f32: Kcur_pos-32 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.48, var = 0.02 | |
| 1.08.260.223 I 01326: f16: cache_k_l32 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.48, var = 0.01 | |
| 1.08.274.877 I 01327: f32: Vcur_normed-32 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.01, max = 15.96, var = 1.00 | |
| 1.08.318.834 I 01328: f16: cache_v_l32 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.00, max = 15.95, var = 0.33 | |
| 1.08.348.105 I 01329: f32: Qcur_pos-32 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.66, avg = 0.01, max = 14.88, var = 1.02 | |
| 1.08.377.396 I 01330: f32: Qcur_pos-32 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.66, avg = 0.01, max = 14.88, var = 1.02 | |
| 1.08.392.023 I 01331: f16: cache_k_l32 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.48, var = 0.02 | |
| 1.08.406.638 I 01332: f16: cache_k_l32 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.48, var = 0.02 | |
| 1.08.421.183 I 01333: f16: cache_v_l32 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.01, max = 15.95, var = 1.00 | |
| 1.08.435.732 I 01334: f16: cache_v_l32 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.01, max = 15.95, var = 1.00 | |
| 1.08.475.533 I 01335: f32: __fattn__-32 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.83, avg = 0.00, max = 15.91, var = 0.25 | |
| 1.08.504.834 I 01336: f32: kqv_out-32 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.83, avg = 0.00, max = 15.91, var = 0.25 | |
| 1.08.561.971 I 01337: f32: node_1336 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.33, avg = -0.01, max = 76.23, var = 1.38 | |
| 1.08.582.326 I 01338: f32: norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.47, avg = -0.01, max = 52.62, var = 1.00 | |
| 1.08.601.959 I 01339: f32: attn_post_norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.47, avg = 0.01, max = 52.62, var = 0.46 | |
| 1.08.621.656 I 01340: f32: attn_out-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -104.04, avg = 0.06, max = 325.48, var = 18.39 | |
| 1.08.641.587 I 01341: f32: norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.52, avg = 0.01, max = 64.93, var = 1.00 | |
| 1.08.661.084 I 01342: f32: ffn_norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.84, avg = -0.00, max = 12.35, var = 0.04 | |
| 1.08.836.320 I 01343: f32: ffn_gate-32 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.54, avg = -0.14, max = 20.00, var = 0.09 | |
| 1.09.012.138 I 01344: f32: ffn_up-32 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.62, avg = -0.00, max = 41.11, var = 0.09 | |
| 1.09.093.449 I 01345: f32: ffn_geglu-32 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -227.91, avg = 0.00, max = 405.27, var = 0.02 | |
| 1.09.209.985 I 01346: f32: ffn_out-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -64.91, avg = 0.00, max = 90.75, var = 0.07 | |
| 1.09.229.851 I 01347: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.70, avg = 0.01, max = 43.40, var = 1.00 | |
| 1.09.249.488 I 01348: f32: ffn_post_norm-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.01, avg = 0.01, max = 57.30, var = 0.57 | |
| 1.09.269.200 I 01349: f32: node_1348 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -119.05, avg = 0.07, max = 377.78, var = 23.72 | |
| 1.09.289.139 I 01350: f32: l_out-32 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -91.61, avg = 0.06, max = 290.71, var = 14.05 | |
| 1.09.309.088 I 01351: f32: norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.36, avg = 0.02, max = 64.41, var = 1.00 | |
| 1.09.328.652 I 01352: f32: attn_norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -868.92, avg = -0.01, max = 158.23, var = 2.21 | |
| 1.09.396.604 I 01353: f32: Qcur-33 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -103.08, avg = 0.03, max = 109.90, var = 4.71 | |
| 1.09.426.803 I 01354: f32: Qcur-33 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -103.08, avg = 0.03, max = 109.90, var = 4.71 | |
| 1.09.457.193 I 01355: f32: norm-33 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.65, avg = 0.01, max = 14.41, var = 1.00 | |
| 1.09.486.816 I 01356: f32: Qcur_normed-33 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.22, avg = 0.01, max = 14.98, var = 1.08 | |
| 1.09.516.840 I 01357: f32: Qcur_pos-33 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.25, avg = 0.01, max = 15.32, var = 1.08 | |
| 1.09.550.819 I 01358: f32: Kcur-33 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -227.56, avg = -0.01, max = 223.41, var = 6.38 | |
| 1.09.565.926 I 01359: f32: Kcur-33 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -227.56, avg = -0.01, max = 223.41, var = 6.38 | |
| 1.09.581.359 I 01360: f32: norm-33 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.26, avg = -0.01, max = 13.55, var = 1.00 | |
| 1.09.596.310 I 01361: f32: Kcur_normed-33 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.63, var = 0.01 | |
| 1.09.611.477 I 01362: f32: Kcur_pos-33 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.63, var = 0.01 | |
| 1.09.645.682 I 01363: f32: Vcur-33 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -563.34, avg = 0.01, max = 559.39, var = 5.90 | |
| 1.09.660.797 I 01364: f32: Vcur-33 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -563.34, avg = 0.01, max = 559.39, var = 5.90 | |
| 1.09.676.122 I 01365: f32: Vcur_normed-33 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.09.690.804 I 01366: f32: Kcur_pos-33 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.63, var = 0.01 | |
| 1.09.734.851 I 01367: f16: cache_k_l33 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.63, var = 0.00 | |
| 1.09.749.511 I 01368: f32: Vcur_normed-33 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.09.793.450 I 01369: f16: cache_v_l33 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.95, var = 0.33 | |
| 1.09.822.712 I 01370: f32: Qcur_pos-33 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.25, avg = 0.01, max = 15.32, var = 1.08 | |
| 1.09.851.973 I 01371: f32: Qcur_pos-33 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.25, avg = 0.01, max = 15.32, var = 1.08 | |
| 1.09.866.589 I 01372: f16: cache_k_l33 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.63, var = 0.01 | |
| 1.09.881.233 I 01373: f16: cache_k_l33 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = -0.00, max = 1.63, var = 0.01 | |
| 1.09.895.804 I 01374: f16: cache_v_l33 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.09.910.373 I 01375: f16: cache_v_l33 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.09.950.216 I 01376: f32: __fattn__-33 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.66, avg = 0.00, max = 15.86, var = 0.25 | |
| 1.09.979.566 I 01377: f32: kqv_out-33 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.66, avg = 0.00, max = 15.86, var = 0.25 | |
| 1.10.036.734 I 01378: f32: node_1377 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.69, avg = -0.00, max = 44.24, var = 0.92 | |
| 1.10.056.637 I 01379: f32: norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.79, avg = -0.00, max = 44.55, var = 1.00 | |
| 1.10.076.287 I 01380: f32: attn_post_norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.05, avg = 0.00, max = 38.11, var = 0.23 | |
| 1.10.096.014 I 01381: f32: attn_out-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -91.18, avg = 0.06, max = 326.87, var = 17.18 | |
| 1.10.115.998 I 01382: f32: norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.48, avg = 0.01, max = 63.92, var = 1.00 | |
| 1.10.135.513 I 01383: f32: ffn_norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -169.64, avg = -0.00, max = 28.49, var = 0.18 | |
| 1.10.311.377 I 01384: f32: ffn_gate-33 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.35, avg = -0.47, max = 54.77, var = 0.45 | |
| 1.10.486.887 I 01385: f32: ffn_up-33 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -42.13, avg = -0.00, max = 45.13, var = 0.25 | |
| 1.10.568.040 I 01386: f32: ffn_geglu-33 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -124.95, avg = 0.00, max = 533.34, var = 0.06 | |
| 1.10.684.590 I 01387: f32: ffn_out-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.29, avg = 0.00, max = 96.68, var = 0.23 | |
| 1.10.704.523 I 01388: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.73, avg = 0.01, max = 39.52, var = 1.00 | |
| 1.10.724.209 I 01389: f32: ffn_post_norm-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.79, avg = 0.01, max = 49.71, var = 0.61 | |
| 1.10.743.913 I 01390: f32: node_1389 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -91.98, avg = 0.07, max = 374.09, var = 22.43 | |
| 1.10.763.897 I 01391: f32: l_out-33 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -73.30, avg = 0.06, max = 298.11, var = 14.24 | |
| 1.10.783.869 I 01392: f32: norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.82, avg = 0.02, max = 63.53, var = 1.00 | |
| 1.10.803.431 I 01393: f32: attn_norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -771.29, avg = -0.02, max = 91.54, var = 1.73 | |
| 1.10.871.119 I 01394: f32: Qcur-34 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -117.17, avg = 0.00, max = 131.18, var = 4.64 | |
| 1.10.901.193 I 01395: f32: Qcur-34 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -117.17, avg = 0.00, max = 131.18, var = 4.64 | |
| 1.10.931.435 I 01396: f32: norm-34 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.61, avg = 0.00, max = 13.76, var = 1.00 | |
| 1.10.961.092 I 01397: f32: Qcur_normed-34 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.61, avg = 0.00, max = 13.76, var = 1.00 | |
| 1.10.991.046 I 01398: f32: Qcur_pos-34 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.61, avg = -0.00, max = 13.79, var = 1.00 | |
| 1.11.024.993 I 01399: f32: Kcur-34 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -204.52, avg = -0.01, max = 171.04, var = 5.84 | |
| 1.11.040.086 I 01400: f32: Kcur-34 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -204.52, avg = -0.01, max = 171.04, var = 5.84 | |
| 1.11.055.511 I 01401: f32: norm-34 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.51, avg = -0.00, max = 13.35, var = 1.00 | |
| 1.11.070.486 I 01402: f32: Kcur_normed-34 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.56, avg = -0.00, max = 1.67, var = 0.02 | |
| 1.11.085.668 I 01403: f32: Kcur_pos-34 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.57, avg = -0.00, max = 1.67, var = 0.02 | |
| 1.11.119.712 I 01404: f32: Vcur-34 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -553.12, avg = 0.01, max = 510.36, var = 5.04 | |
| 1.11.134.816 I 01405: f32: Vcur-34 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -553.12, avg = 0.01, max = 510.36, var = 5.04 | |
| 1.11.150.428 I 01406: f32: Vcur_normed-34 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.11.165.565 I 01407: f32: Kcur_pos-34 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.57, avg = -0.00, max = 1.67, var = 0.02 | |
| 1.11.209.610 I 01408: f16: cache_k_l34 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.57, avg = -0.00, max = 1.67, var = 0.01 | |
| 1.11.224.274 I 01409: f32: Vcur_normed-34 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.11.268.204 I 01410: f16: cache_v_l34 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.00, max = 15.95, var = 0.33 | |
| 1.11.297.500 I 01411: f32: Qcur_pos-34 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.61, avg = -0.00, max = 13.79, var = 1.00 | |
| 1.11.327.414 I 01412: f32: Qcur_pos-34 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.61, avg = -0.00, max = 13.79, var = 1.00 | |
| 1.11.342.072 I 01413: f16: cache_k_l34 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.57, avg = -0.00, max = 1.67, var = 0.02 | |
| 1.11.356.682 I 01414: f16: cache_k_l34 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.57, avg = -0.00, max = 1.67, var = 0.02 | |
| 1.11.371.221 I 01415: f16: cache_v_l34 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.11.385.762 I 01416: f16: cache_v_l34 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = 0.01, max = 15.95, var = 1.00 | |
| 1.11.425.862 I 01417: f32: __fattn__-34 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.01, max = 15.95, var = 0.22 | |
| 1.11.455.179 I 01418: f32: kqv_out-34 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.96, avg = -0.01, max = 15.95, var = 0.22 | |
| 1.11.512.243 I 01419: f32: node_1418 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.07, avg = -0.01, max = 36.57, var = 0.98 | |
| 1.11.532.596 I 01420: f32: norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.74, avg = -0.01, max = 42.54, var = 1.00 | |
| 1.11.552.181 I 01421: f32: attn_post_norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.56, avg = 0.00, max = 42.04, var = 0.24 | |
| 1.11.571.886 I 01422: f32: attn_out-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -72.57, avg = 0.06, max = 340.15, var = 17.16 | |
| 1.11.591.873 I 01423: f32: norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.48, avg = 0.01, max = 63.00, var = 1.00 | |
| 1.11.611.383 I 01424: f32: ffn_norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -259.73, avg = -0.01, max = 38.58, var = 0.35 | |
| 1.11.787.076 I 01425: f32: ffn_gate-34 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.63, avg = -0.96, max = 30.41, var = 0.96 | |
| 1.11.963.080 I 01426: f32: ffn_up-34 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -60.08, avg = -0.00, max = 71.62, var = 0.59 | |
| 1.12.044.198 I 01427: f32: ffn_geglu-34 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -138.86, avg = -0.00, max = 652.37, var = 0.10 | |
| 1.12.160.916 I 01428: f32: ffn_out-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.72, avg = 0.01, max = 62.91, var = 0.47 | |
| 1.12.180.756 I 01429: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.34, avg = 0.01, max = 52.70, var = 1.00 | |
| 1.12.200.370 I 01430: f32: ffn_post_norm-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -45.06, avg = 0.01, max = 71.23, var = 0.94 | |
| 1.12.220.117 I 01431: f32: node_1430 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -79.60, avg = 0.08, max = 391.89, var = 24.07 | |
| 1.12.240.041 I 01432: f32: l_out-34 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -60.95, avg = 0.06, max = 300.04, var = 14.11 | |
| 1.12.259.974 I 01433: f32: norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.50, avg = 0.02, max = 63.49, var = 1.00 | |
| 1.12.279.507 I 01434: f32: attn_norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -550.31, avg = -0.02, max = 87.21, var = 1.17 | |
| 1.12.414.165 I 01435: f32: Qcur-35 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.00, avg = -0.03, max = 121.24, var = 1.97 | |
| 1.12.472.865 I 01436: f32: Qcur-35 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.00, avg = -0.03, max = 121.24, var = 1.97 | |
| 1.12.533.878 I 01437: f32: norm-35 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.76, avg = -0.02, max = 20.43, var = 1.00 | |
| 1.12.593.184 I 01438: f32: Qcur_normed-35 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.82, avg = -0.02, max = 19.55, var = 0.92 | |
| 1.12.652.703 I 01439: f32: Qcur_pos-35 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.82, avg = -0.02, max = 19.55, var = 0.92 | |
| 1.12.670.127 I 01440: f32: Vcur-35 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -168.67, avg = 0.02, max = 175.09, var = 2.05 | |
| 1.12.677.676 I 01441: f32: Vcur-35 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -168.67, avg = 0.02, max = 175.09, var = 2.05 | |
| 1.12.685.725 I 01442: f32: norm-35 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.62, avg = 0.01, max = 21.16, var = 1.00 | |
| 1.12.693.365 I 01443: f32: Kcur_normed-35 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.38, var = 0.00 | |
| 1.12.701.144 I 01444: f32: Kcur_pos-35 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.38, var = 0.00 | |
| 1.12.708.489 I 01445: f32: Vcur-35 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -168.67, avg = 0.02, max = 175.09, var = 2.05 | |
| 1.12.716.388 I 01446: f32: Vcur_normed-35 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.62, avg = 0.01, max = 21.16, var = 1.00 | |
| 1.12.723.751 I 01447: f32: Kcur_pos-35 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.38, var = 0.00 | |
| 1.12.782.172 I 01448: f16: cache_k_l35 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.38, var = 0.00 | |
| 1.12.789.503 I 01449: f32: Vcur_normed-35 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.62, avg = 0.01, max = 21.16, var = 1.00 | |
| 1.12.847.887 I 01450: f16: cache_v_l35 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.61, avg = 0.00, max = 21.16, var = 0.12 | |
| 1.12.906.443 I 01451: f32: Qcur_pos-35 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.82, avg = -0.02, max = 19.55, var = 0.92 | |
| 1.12.964.952 I 01452: f32: Qcur_pos-35 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.82, avg = -0.02, max = 19.55, var = 0.92 | |
| 1.12.972.274 I 01453: f16: cache_k_l35 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.38, var = 0.00 | |
| 1.12.979.623 I 01454: f16: cache_k_l35 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.38, var = 0.00 | |
| 1.12.986.895 I 01455: f16: cache_v_l35 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.61, avg = 0.01, max = 21.16, var = 1.00 | |
| 1.12.994.175 I 01456: f16: cache_v_l35 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.61, avg = 0.01, max = 21.16, var = 1.00 | |
| 1.13.072.214 I 01457: f32: __fattn__-35 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.41, avg = 0.01, max = 19.58, var = 0.29 | |
| 1.13.130.820 I 01458: f32: kqv_out-35 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.41, avg = 0.01, max = 19.58, var = 0.29 | |
| 1.13.224.436 I 01459: f32: node_1458 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.04, avg = 0.04, max = 65.75, var = 2.36 | |
| 1.13.244.404 I 01460: f32: norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.06, avg = 0.02, max = 49.36, var = 1.00 | |
| 1.13.264.031 I 01461: f32: attn_post_norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.19, avg = 0.01, max = 65.17, var = 0.41 | |
| 1.13.283.730 I 01462: f32: attn_out-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -69.38, avg = 0.07, max = 351.01, var = 18.30 | |
| 1.13.303.665 I 01463: f32: norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.83, avg = 0.02, max = 64.06, var = 1.00 | |
| 1.13.323.206 I 01464: f32: ffn_norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -236.46, avg = -0.00, max = 23.94, var = 0.20 | |
| 1.13.499.140 I 01465: f32: ffn_gate-35 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.58, avg = -0.45, max = 21.98, var = 0.50 | |
| 1.13.673.999 I 01466: f32: ffn_up-35 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.48, avg = 0.00, max = 44.29, var = 0.31 | |
| 1.13.756.282 I 01467: f32: ffn_geglu-35 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -291.29, avg = -0.00, max = 293.86, var = 0.04 | |
| 1.13.872.991 I 01468: f32: ffn_out-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.67, avg = 0.01, max = 83.48, var = 0.23 | |
| 1.13.892.881 I 01469: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.11, avg = 0.02, max = 57.49, var = 1.00 | |
| 1.13.912.320 I 01470: f32: ffn_post_norm-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.91, avg = 0.03, max = 98.82, var = 1.65 | |
| 1.13.932.031 I 01471: f32: node_1470 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -78.79, avg = 0.09, max = 444.06, var = 28.60 | |
| 1.13.952.042 I 01472: f32: l_out-35 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.55, avg = 0.06, max = 301.82, var = 13.21 | |
| 1.13.972.011 I 01473: f32: norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.02, max = 64.64, var = 1.00 | |
| 1.13.991.543 I 01474: f32: attn_norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -737.34, avg = -0.00, max = 75.43, var = 1.47 | |
| 1.14.059.012 I 01475: f32: Qcur-36 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.91, avg = 0.03, max = 57.31, var = 2.34 | |
| 1.14.089.215 I 01476: f32: Qcur-36 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.91, avg = 0.03, max = 57.31, var = 2.34 | |
| 1.14.119.507 I 01477: f32: norm-36 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.25, avg = 0.02, max = 14.70, var = 1.00 | |
| 1.14.149.117 I 01478: f32: Qcur_normed-36 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.75, avg = 0.02, max = 14.18, var = 0.93 | |
| 1.14.179.287 I 01479: f32: Qcur_pos-36 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.06, avg = 0.02, max = 14.25, var = 0.93 | |
| 1.14.213.474 I 01480: f32: Kcur-36 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -158.13, avg = -0.00, max = 162.14, var = 3.40 | |
| 1.14.228.584 I 01481: f32: Kcur-36 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -158.13, avg = -0.00, max = 162.14, var = 3.40 | |
| 1.14.243.936 I 01482: f32: norm-36 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.97, avg = -0.01, max = 12.97, var = 1.00 | |
| 1.14.258.898 I 01483: f32: Kcur_normed-36 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.56, avg = -0.00, max = 1.68, var = 0.02 | |
| 1.14.274.063 I 01484: f32: Kcur_pos-36 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.56, avg = -0.00, max = 1.68, var = 0.02 | |
| 1.14.308.085 I 01485: f32: Vcur-36 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -474.66, avg = -0.00, max = 471.16, var = 3.74 | |
| 1.14.323.200 I 01486: f32: Vcur-36 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -474.66, avg = -0.00, max = 471.16, var = 3.74 | |
| 1.14.338.734 I 01487: f32: Vcur_normed-36 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.93, var = 1.00 | |
| 1.14.353.502 I 01488: f32: Kcur_pos-36 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.56, avg = -0.00, max = 1.68, var = 0.02 | |
| 1.14.397.542 I 01489: f16: cache_k_l36 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.68, var = 0.01 | |
| 1.14.412.191 I 01490: f32: Vcur_normed-36 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.93, var = 1.00 | |
| 1.14.456.120 I 01491: f16: cache_v_l36 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.94, var = 0.33 | |
| 1.14.485.415 I 01492: f32: Qcur_pos-36 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.06, avg = 0.02, max = 14.25, var = 0.93 | |
| 1.14.514.714 I 01493: f32: Qcur_pos-36 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.06, avg = 0.02, max = 14.25, var = 0.93 | |
| 1.14.529.325 I 01494: f16: cache_k_l36 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.68, var = 0.02 | |
| 1.14.543.941 I 01495: f16: cache_k_l36 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.68, var = 0.02 | |
| 1.14.558.487 I 01496: f16: cache_v_l36 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.94, var = 1.00 | |
| 1.14.573.037 I 01497: f16: cache_v_l36 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.94, var = 1.00 | |
| 1.14.613.184 I 01498: f32: __fattn__-36 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = 0.00, max = 15.94, var = 0.24 | |
| 1.14.642.528 I 01499: f32: kqv_out-36 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = 0.00, max = 15.94, var = 0.24 | |
| 1.14.699.696 I 01500: f32: node_1499 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.96, avg = 0.00, max = 32.20, var = 0.70 | |
| 1.14.719.597 I 01501: f32: norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.10, avg = 0.00, max = 34.52, var = 1.00 | |
| 1.14.739.162 I 01502: f32: attn_post_norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.06, avg = 0.00, max = 27.51, var = 0.17 | |
| 1.14.758.866 I 01503: f32: attn_out-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.42, avg = 0.07, max = 329.33, var = 15.13 | |
| 1.14.778.838 I 01504: f32: norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.16, avg = 0.02, max = 64.30, var = 1.00 | |
| 1.14.798.404 I 01505: f32: ffn_norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -78.28, avg = -0.00, max = 20.92, var = 0.06 | |
| 1.14.974.710 I 01506: f32: ffn_gate-36 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.51, avg = -0.19, max = 31.28, var = 0.14 | |
| 1.15.149.541 I 01507: f32: ffn_up-36 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.30, avg = 0.00, max = 18.49, var = 0.10 | |
| 1.15.230.709 I 01508: f32: ffn_geglu-36 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.88, avg = 0.00, max = 151.02, var = 0.01 | |
| 1.15.347.502 I 01509: f32: ffn_out-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.11, avg = 0.00, max = 28.12, var = 0.03 | |
| 1.15.367.512 I 01510: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.43, avg = 0.01, max = 40.44, var = 1.00 | |
| 1.15.387.213 I 01511: f32: ffn_post_norm-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.95, avg = 0.01, max = 52.77, var = 0.71 | |
| 1.15.406.905 I 01512: f32: node_1511 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -67.04, avg = 0.08, max = 382.10, var = 19.47 | |
| 1.15.426.904 I 01513: f32: l_out-36 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.16, avg = 0.06, max = 302.99, var = 12.24 | |
| 1.15.446.861 I 01514: f32: norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.73, avg = 0.02, max = 64.75, var = 1.00 | |
| 1.15.466.382 I 01515: f32: attn_norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -624.36, avg = -0.01, max = 107.73, var = 1.05 | |
| 1.15.533.826 I 01516: f32: Qcur-37 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.04, avg = 0.01, max = 42.85, var = 1.70 | |
| 1.15.564.031 I 01517: f32: Qcur-37 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.04, avg = 0.01, max = 42.85, var = 1.70 | |
| 1.15.595.063 I 01518: f32: norm-37 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.73, avg = 0.00, max = 13.76, var = 1.00 | |
| 1.15.625.013 I 01519: f32: Qcur_normed-37 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.51, avg = 0.00, max = 13.54, var = 0.97 | |
| 1.15.654.996 I 01520: f32: Qcur_pos-37 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.51, avg = -0.00, max = 13.51, var = 0.97 | |
| 1.15.689.284 I 01521: f32: Kcur-37 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -162.76, avg = 0.01, max = 168.48, var = 2.50 | |
| 1.15.704.397 I 01522: f32: Kcur-37 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -162.76, avg = 0.01, max = 168.48, var = 2.50 | |
| 1.15.719.734 I 01523: f32: norm-37 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.13, avg = 0.01, max = 13.89, var = 1.00 | |
| 1.15.734.717 I 01524: f32: Kcur_normed-37 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.67, avg = 0.00, max = 1.76, var = 0.02 | |
| 1.15.749.864 I 01525: f32: Kcur_pos-37 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.67, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.15.783.868 I 01526: f32: Vcur-37 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -482.94, avg = -0.00, max = 440.37, var = 3.84 | |
| 1.15.798.983 I 01527: f32: Vcur-37 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -482.94, avg = -0.00, max = 440.37, var = 3.84 | |
| 1.15.814.267 I 01528: f32: Vcur_normed-37 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = -0.01, max = 15.96, var = 1.00 | |
| 1.15.828.960 I 01529: f32: Kcur_pos-37 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.67, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.15.873.043 I 01530: f16: cache_k_l37 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.67, avg = -0.00, max = 1.76, var = 0.01 | |
| 1.15.889.034 I 01531: f32: Vcur_normed-37 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = -0.01, max = 15.96, var = 1.00 | |
| 1.15.932.845 I 01532: f16: cache_v_l37 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = -0.00, max = 15.96, var = 0.33 | |
| 1.15.962.752 I 01533: f32: Qcur_pos-37 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.51, avg = -0.00, max = 13.51, var = 0.97 | |
| 1.15.992.949 I 01534: f32: Qcur_pos-37 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.51, avg = -0.00, max = 13.51, var = 0.97 | |
| 1.16.007.601 I 01535: f16: cache_k_l37 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.67, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.16.022.214 I 01536: f16: cache_k_l37 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.67, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.16.036.763 I 01537: f16: cache_v_l37 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = -0.01, max = 15.96, var = 1.00 | |
| 1.16.051.311 I 01538: f16: cache_v_l37 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.90, avg = -0.01, max = 15.96, var = 1.00 | |
| 1.16.092.268 I 01539: f32: __fattn__-37 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.41, avg = -0.00, max = 15.96, var = 0.23 | |
| 1.16.121.957 I 01540: f32: kqv_out-37 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.41, avg = -0.00, max = 15.96, var = 0.23 | |
| 1.16.185.457 I 01541: f32: node_1540 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.93, avg = -0.00, max = 32.22, var = 0.66 | |
| 1.16.205.352 I 01542: f32: norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.89, avg = -0.01, max = 32.31, var = 1.00 | |
| 1.16.224.962 I 01543: f32: attn_post_norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.40, avg = 0.00, max = 24.99, var = 0.13 | |
| 1.16.244.646 I 01544: f32: attn_out-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.13, avg = 0.06, max = 321.22, var = 13.58 | |
| 1.16.264.617 I 01545: f32: norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.87, avg = 0.02, max = 64.81, var = 1.00 | |
| 1.16.284.127 I 01546: f32: ffn_norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -80.56, avg = -0.00, max = 14.92, var = 0.07 | |
| 1.16.459.210 I 01547: f32: ffn_gate-37 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.60, avg = -0.20, max = 40.22, var = 0.15 | |
| 1.16.635.074 I 01548: f32: ffn_up-37 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.17, avg = -0.00, max = 39.56, var = 0.11 | |
| 1.16.716.274 I 01549: f32: ffn_geglu-37 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -50.47, avg = 0.00, max = 178.63, var = 0.01 | |
| 1.16.832.874 I 01550: f32: ffn_out-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.11, avg = 0.00, max = 17.82, var = 0.04 | |
| 1.16.852.790 I 01551: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.34, avg = 0.01, max = 45.66, var = 1.00 | |
| 1.16.872.442 I 01552: f32: ffn_post_norm-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.94, avg = 0.02, max = 54.94, var = 0.65 | |
| 1.16.892.250 I 01553: f32: node_1552 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.13, avg = 0.08, max = 373.33, var = 17.56 | |
| 1.16.912.755 I 01554: f32: l_out-37 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.05, avg = 0.06, max = 303.33, var = 11.59 | |
| 1.16.932.812 I 01555: f32: norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.85, avg = 0.02, max = 65.86, var = 1.00 | |
| 1.16.952.361 I 01556: f32: attn_norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -578.18, avg = 0.00, max = 68.46, var = 0.81 | |
| 1.17.035.890 I 01557: f32: Qcur-38 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.12, avg = 0.01, max = 27.86, var = 1.61 | |
| 1.17.066.116 I 01558: f32: Qcur-38 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.12, avg = 0.01, max = 27.86, var = 1.61 | |
| 1.17.096.561 I 01559: f32: norm-38 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.59, avg = 0.00, max = 14.53, var = 1.00 | |
| 1.17.126.031 I 01560: f32: Qcur_normed-38 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.12, avg = 0.00, max = 15.09, var = 1.08 | |
| 1.17.156.294 I 01561: f32: Qcur_pos-38 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.66, avg = -0.01, max = 15.09, var = 1.08 | |
| 1.17.191.252 I 01562: f32: Kcur-38 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -126.88, avg = -0.00, max = 116.98, var = 1.86 | |
| 1.17.206.364 I 01563: f32: Kcur-38 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -126.88, avg = -0.00, max = 116.98, var = 1.86 | |
| 1.17.221.748 I 01564: f32: norm-38 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.39, avg = -0.00, max = 14.41, var = 1.00 | |
| 1.17.236.725 I 01565: f32: Kcur_normed-38 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.61, avg = -0.00, max = 1.73, var = 0.01 | |
| 1.17.252.039 I 01566: f32: Kcur_pos-38 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.61, avg = 0.00, max = 1.74, var = 0.01 | |
| 1.17.288.036 I 01567: f32: Vcur-38 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -436.77, avg = -0.02, max = 403.27, var = 2.56 | |
| 1.17.303.148 I 01568: f32: Vcur-38 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -436.77, avg = -0.02, max = 403.27, var = 2.56 | |
| 1.17.318.374 I 01569: f32: Vcur_normed-38 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.02, max = 15.90, var = 1.00 | |
| 1.17.333.064 I 01570: f32: Kcur_pos-38 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.61, avg = 0.00, max = 1.74, var = 0.01 | |
| 1.17.377.133 I 01571: f16: cache_k_l38 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.61, avg = 0.00, max = 1.74, var = 0.00 | |
| 1.17.391.913 I 01572: f32: Vcur_normed-38 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.02, max = 15.90, var = 1.00 | |
| 1.17.435.844 I 01573: f16: cache_v_l38 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.01, max = 15.90, var = 0.33 | |
| 1.17.465.109 I 01574: f32: Qcur_pos-38 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.66, avg = -0.01, max = 15.09, var = 1.08 | |
| 1.17.494.424 I 01575: f32: Qcur_pos-38 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.66, avg = -0.01, max = 15.09, var = 1.08 | |
| 1.17.509.022 I 01576: f16: cache_k_l38 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.61, avg = 0.00, max = 1.74, var = 0.01 | |
| 1.17.523.643 I 01577: f16: cache_k_l38 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.61, avg = 0.00, max = 1.74, var = 0.01 | |
| 1.17.538.182 I 01578: f16: cache_v_l38 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.02, max = 15.90, var = 1.00 | |
| 1.17.552.719 I 01579: f16: cache_v_l38 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.02, max = 15.90, var = 1.00 | |
| 1.17.593.171 I 01580: f32: __fattn__-38 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.80, avg = -0.00, max = 15.89, var = 0.25 | |
| 1.17.622.489 I 01581: f32: kqv_out-38 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.80, avg = -0.00, max = 15.89, var = 0.25 | |
| 1.17.681.289 I 01582: f32: node_1581 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.77, avg = 0.00, max = 29.34, var = 0.52 | |
| 1.17.701.173 I 01583: f32: norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.90, avg = 0.00, max = 33.76, var = 1.00 | |
| 1.17.720.796 I 01584: f32: attn_post_norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.91, avg = -0.00, max = 25.14, var = 0.17 | |
| 1.17.740.534 I 01585: f32: attn_out-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -52.58, avg = 0.06, max = 320.01, var = 12.92 | |
| 1.17.760.508 I 01586: f32: norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.31, avg = 0.02, max = 65.91, var = 1.00 | |
| 1.17.780.051 I 01587: f32: ffn_norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -83.36, avg = -0.00, max = 15.99, var = 0.06 | |
| 1.17.955.191 I 01588: f32: ffn_gate-38 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.86, avg = -0.18, max = 20.55, var = 0.13 | |
| 1.18.132.089 I 01589: f32: ffn_up-38 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.05, avg = -0.00, max = 36.75, var = 0.10 | |
| 1.18.213.745 I 01590: f32: ffn_geglu-38 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.12, avg = -0.00, max = 67.10, var = 0.00 | |
| 1.18.331.012 I 01591: f32: ffn_out-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.44, avg = 0.00, max = 16.65, var = 0.03 | |
| 1.18.350.871 I 01592: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.83, avg = 0.01, max = 35.78, var = 1.00 | |
| 1.18.370.545 I 01593: f32: ffn_post_norm-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.57, avg = 0.01, max = 46.40, var = 0.69 | |
| 1.18.390.254 I 01594: f32: node_1593 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -54.37, avg = 0.07, max = 361.77, var = 15.58 | |
| 1.18.410.678 I 01595: f32: l_out-38 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -43.54, avg = 0.06, max = 289.70, var = 9.99 | |
| 1.18.430.700 I 01596: f32: norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.78, avg = 0.02, max = 67.92, var = 1.00 | |
| 1.18.450.238 I 01597: f32: attn_norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -683.38, avg = -0.00, max = 68.61, var = 1.07 | |
| 1.18.518.409 I 01598: f32: Qcur-39 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.88, avg = -0.01, max = 29.53, var = 2.13 | |
| 1.18.548.613 I 01599: f32: Qcur-39 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.88, avg = -0.01, max = 29.53, var = 2.13 | |
| 1.18.579.069 I 01600: f32: norm-39 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.89, avg = -0.01, max = 14.10, var = 1.00 | |
| 1.18.608.704 I 01601: f32: Qcur_normed-39 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.54, avg = -0.01, max = 14.76, var = 1.10 | |
| 1.18.638.932 I 01602: f32: Qcur_pos-39 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.57, avg = -0.01, max = 14.76, var = 1.10 | |
| 1.18.673.472 I 01603: f32: Kcur-39 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -135.55, avg = 0.02, max = 140.09, var = 2.64 | |
| 1.18.688.577 I 01604: f32: Kcur-39 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -135.55, avg = 0.02, max = 140.09, var = 2.64 | |
| 1.18.703.962 I 01605: f32: norm-39 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.35, avg = 0.01, max = 12.55, var = 1.00 | |
| 1.18.719.075 I 01606: f32: Kcur_normed-39 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.48, avg = 0.00, max = 1.50, var = 0.01 | |
| 1.18.734.453 I 01607: f32: Kcur_pos-39 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.48, avg = 0.00, max = 1.53, var = 0.01 | |
| 1.18.769.617 I 01608: f32: Vcur-39 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -350.18, avg = 0.00, max = 374.06, var = 2.63 | |
| 1.18.784.730 I 01609: f32: Vcur-39 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -350.18, avg = 0.00, max = 374.06, var = 2.63 | |
| 1.18.800.025 I 01610: f32: Vcur_normed-39 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.00, max = 15.93, var = 1.00 | |
| 1.18.814.720 I 01611: f32: Kcur_pos-39 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.48, avg = 0.00, max = 1.53, var = 0.01 | |
| 1.18.858.794 I 01612: f16: cache_k_l39 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.48, avg = 0.00, max = 1.53, var = 0.00 | |
| 1.18.873.452 I 01613: f32: Vcur_normed-39 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.00, max = 15.93, var = 1.00 | |
| 1.18.917.447 I 01614: f16: cache_v_l39 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.00, max = 15.92, var = 0.33 | |
| 1.18.946.737 I 01615: f32: Qcur_pos-39 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.57, avg = -0.01, max = 14.76, var = 1.10 | |
| 1.18.976.567 I 01616: f32: Qcur_pos-39 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.57, avg = -0.01, max = 14.76, var = 1.10 | |
| 1.18.991.236 I 01617: f16: cache_k_l39 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.48, avg = 0.00, max = 1.53, var = 0.01 | |
| 1.19.005.859 I 01618: f16: cache_k_l39 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.48, avg = 0.00, max = 1.53, var = 0.01 | |
| 1.19.020.409 I 01619: f16: cache_v_l39 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.00, max = 15.92, var = 1.00 | |
| 1.19.034.958 I 01620: f16: cache_v_l39 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.93, avg = 0.00, max = 15.92, var = 1.00 | |
| 1.19.077.473 I 01621: f32: __fattn__-39 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.70, var = 0.25 | |
| 1.19.107.035 I 01622: f32: kqv_out-39 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.91, avg = -0.00, max = 15.70, var = 0.25 | |
| 1.19.165.086 I 01623: f32: node_1622 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.19, avg = 0.01, max = 25.64, var = 0.82 | |
| 1.19.185.470 I 01624: f32: norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.93, avg = 0.01, max = 23.58, var = 1.00 | |
| 1.19.205.117 I 01625: f32: attn_post_norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.31, avg = 0.00, max = 25.44, var = 0.25 | |
| 1.19.224.875 I 01626: f32: attn_out-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -46.92, avg = 0.06, max = 315.14, var = 11.35 | |
| 1.19.244.582 I 01627: f32: norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.10, avg = 0.02, max = 68.61, var = 1.00 | |
| 1.19.264.134 I 01628: f32: ffn_norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.62, avg = 0.00, max = 16.19, var = 0.07 | |
| 1.19.441.580 I 01629: f32: ffn_gate-39 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.48, avg = -0.26, max = 15.25, var = 0.19 | |
| 1.19.618.264 I 01630: f32: ffn_up-39 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.09, avg = -0.00, max = 33.17, var = 0.13 | |
| 1.19.699.394 I 01631: f32: ffn_geglu-39 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.26, avg = 0.00, max = 225.35, var = 0.01 | |
| 1.19.831.490 I 01632: f32: ffn_out-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.61, avg = -0.00, max = 29.97, var = 0.06 | |
| 1.19.851.415 I 01633: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.21, avg = -0.01, max = 37.70, var = 1.00 | |
| 1.19.870.987 I 01634: f32: ffn_post_norm-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -72.11, avg = -0.00, max = 65.97, var = 1.52 | |
| 1.19.890.502 I 01635: f32: node_1634 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -50.35, avg = 0.06, max = 366.40, var = 14.66 | |
| 1.19.910.498 I 01636: f32: l_out-39 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.01, avg = 0.04, max = 254.76, var = 7.09 | |
| 1.19.930.521 I 01637: f32: norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.68, avg = 0.01, max = 71.97, var = 1.00 | |
| 1.19.949.995 I 01638: f32: attn_norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -561.74, avg = -0.00, max = 38.98, var = 0.65 | |
| 1.20.017.882 I 01639: f32: Qcur-40 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.82, avg = 0.01, max = 30.78, var = 1.83 | |
| 1.20.047.884 I 01640: f32: Qcur-40 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.82, avg = 0.01, max = 30.78, var = 1.83 | |
| 1.20.078.082 I 01641: f32: norm-40 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.00, avg = 0.00, max = 13.58, var = 1.00 | |
| 1.20.107.539 I 01642: f32: Qcur_normed-40 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.70, avg = 0.00, max = 13.26, var = 0.95 | |
| 1.20.137.523 I 01643: f32: Qcur_pos-40 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.70, avg = 0.01, max = 13.26, var = 0.95 | |
| 1.20.172.067 I 01644: f32: Kcur-40 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -120.73, avg = 0.02, max = 121.14, var = 1.74 | |
| 1.20.187.054 I 01645: f32: Kcur-40 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -120.73, avg = 0.02, max = 121.14, var = 1.74 | |
| 1.20.202.442 I 01646: f32: norm-40 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.74, avg = 0.02, max = 12.73, var = 1.00 | |
| 1.20.217.425 I 01647: f32: Kcur_normed-40 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.76, avg = 0.00, max = 1.63, var = 0.02 | |
| 1.20.232.719 I 01648: f32: Kcur_pos-40 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.76, avg = 0.00, max = 1.63, var = 0.02 | |
| 1.20.269.384 I 01649: f32: Vcur-40 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -234.79, avg = -0.00, max = 222.44, var = 1.29 | |
| 1.20.284.496 I 01650: f32: Vcur-40 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -234.79, avg = -0.00, max = 222.44, var = 1.29 | |
| 1.20.299.823 I 01651: f32: Vcur_normed-40 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.94, avg = -0.00, max = 15.83, var = 1.00 | |
| 1.20.314.512 I 01652: f32: Kcur_pos-40 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.76, avg = 0.00, max = 1.63, var = 0.02 | |
| 1.20.358.584 I 01653: f16: cache_k_l40 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.76, avg = 0.00, max = 1.63, var = 0.01 | |
| 1.20.373.244 I 01654: f32: Vcur_normed-40 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.94, avg = -0.00, max = 15.83, var = 1.00 | |
| 1.20.417.193 I 01655: f16: cache_v_l40 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.83, var = 0.33 | |
| 1.20.446.468 I 01656: f32: Qcur_pos-40 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.70, avg = 0.01, max = 13.26, var = 0.95 | |
| 1.20.475.750 I 01657: f32: Qcur_pos-40 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.70, avg = 0.01, max = 13.26, var = 0.95 | |
| 1.20.490.380 I 01658: f16: cache_k_l40 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.76, avg = 0.00, max = 1.63, var = 0.02 | |
| 1.20.504.998 I 01659: f16: cache_k_l40 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.76, avg = 0.00, max = 1.63, var = 0.02 | |
| 1.20.519.553 I 01660: f16: cache_v_l40 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.83, var = 1.00 | |
| 1.20.534.101 I 01661: f16: cache_v_l40 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.83, var = 1.00 | |
| 1.20.573.934 I 01662: f32: __fattn__-40 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.80, var = 0.29 | |
| 1.20.603.257 I 01663: f32: kqv_out-40 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = -0.00, max = 15.80, var = 0.29 | |
| 1.20.660.323 I 01664: f32: node_1663 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.14, avg = 0.00, max = 23.79, var = 0.73 | |
| 1.20.680.372 I 01665: f32: norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.54, avg = 0.00, max = 31.96, var = 1.00 | |
| 1.20.699.912 I 01666: f32: attn_post_norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.54, avg = -0.00, max = 35.20, var = 0.51 | |
| 1.20.719.704 I 01667: f32: attn_out-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.23, avg = 0.04, max = 280.14, var = 8.68 | |
| 1.20.739.628 I 01668: f32: norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.53, avg = 0.01, max = 71.55, var = 1.00 | |
| 1.20.759.160 I 01669: f32: ffn_norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.28, avg = 0.00, max = 42.54, var = 0.28 | |
| 1.20.934.499 I 01670: f32: ffn_gate-40 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.54, avg = -0.42, max = 16.12, var = 0.43 | |
| 1.21.109.561 I 01671: f32: ffn_up-40 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.91, avg = 0.00, max = 22.85, var = 0.28 | |
| 1.21.190.880 I 01672: f32: ffn_geglu-40 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -128.73, avg = 0.00, max = 258.04, var = 0.03 | |
| 1.21.307.701 I 01673: f32: ffn_out-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.10, avg = 0.01, max = 21.20, var = 0.25 | |
| 1.21.327.673 I 01674: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -44.17, avg = 0.01, max = 51.77, var = 1.00 | |
| 1.21.347.334 I 01675: f32: ffn_post_norm-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -102.15, avg = 0.02, max = 119.71, var = 2.30 | |
| 1.21.367.041 I 01676: f32: node_1675 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -121.54, avg = 0.06, max = 399.73, var = 16.03 | |
| 1.21.387.031 I 01677: f32: l_out-40 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.54, avg = 0.04, max = 245.15, var = 6.03 | |
| 1.21.407.010 I 01678: f32: norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.29, avg = 0.02, max = 71.33, var = 1.00 | |
| 1.21.426.553 I 01679: f32: attn_norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -435.54, avg = -0.00, max = 51.97, var = 0.38 | |
| 1.21.561.785 I 01680: f32: Qcur-41 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -52.39, avg = 0.01, max = 53.43, var = 0.56 | |
| 1.21.620.750 I 01681: f32: Qcur-41 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -52.39, avg = 0.01, max = 53.43, var = 0.56 | |
| 1.21.681.655 I 01682: f32: norm-41 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.79, avg = 0.02, max = 21.83, var = 1.00 | |
| 1.21.740.946 I 01683: f32: Qcur_normed-41 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.98, avg = 0.02, max = 20.97, var = 0.92 | |
| 1.21.800.551 I 01684: f32: Qcur_pos-41 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.98, avg = 0.02, max = 20.97, var = 0.92 | |
| 1.21.818.231 I 01685: f32: Vcur-41 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.91, avg = 0.00, max = 103.07, var = 0.54 | |
| 1.21.825.785 I 01686: f32: Vcur-41 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.91, avg = 0.00, max = 103.07, var = 0.54 | |
| 1.21.833.662 I 01687: f32: norm-41 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.48, avg = 0.00, max = 21.97, var = 1.00 | |
| 1.21.841.319 I 01688: f32: Kcur_normed-41 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.40, avg = 0.00, max = 1.43, var = 0.00 | |
| 1.21.848.887 I 01689: f32: Kcur_pos-41 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.40, avg = 0.00, max = 1.43, var = 0.00 | |
| 1.21.856.234 I 01690: f32: Vcur-41 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.91, avg = 0.00, max = 103.07, var = 0.54 | |
| 1.21.864.148 I 01691: f32: Vcur_normed-41 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.48, avg = 0.00, max = 21.97, var = 1.00 | |
| 1.21.871.511 I 01692: f32: Kcur_pos-41 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.40, avg = 0.00, max = 1.43, var = 0.00 | |
| 1.21.929.867 I 01693: f16: cache_k_l41 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.40, avg = 0.00, max = 1.43, var = 0.00 | |
| 1.21.937.195 I 01694: f32: Vcur_normed-41 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.48, avg = 0.00, max = 21.97, var = 1.00 | |
| 1.21.995.581 I 01695: f16: cache_v_l41 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.48, avg = 0.00, max = 21.97, var = 0.12 | |
| 1.22.054.098 I 01696: f32: Qcur_pos-41 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.98, avg = 0.02, max = 20.97, var = 0.92 | |
| 1.22.112.616 I 01697: f32: Qcur_pos-41 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.98, avg = 0.02, max = 20.97, var = 0.92 | |
| 1.22.119.944 I 01698: f16: cache_k_l41 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.40, avg = 0.00, max = 1.43, var = 0.00 | |
| 1.22.127.277 I 01699: f16: cache_k_l41 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.40, avg = 0.00, max = 1.43, var = 0.00 | |
| 1.22.134.552 I 01700: f16: cache_v_l41 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.48, avg = 0.00, max = 21.97, var = 1.00 | |
| 1.22.141.845 I 01701: f16: cache_v_l41 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.48, avg = 0.00, max = 21.97, var = 1.00 | |
| 1.22.219.685 I 01702: f32: __fattn__-41 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.20, avg = 0.01, max = 21.97, var = 0.40 | |
| 1.22.278.312 I 01703: f32: kqv_out-41 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.20, avg = 0.01, max = 21.97, var = 0.40 | |
| 1.22.372.009 I 01704: f32: node_1703 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.61, avg = 0.01, max = 60.15, var = 1.37 | |
| 1.22.391.941 I 01705: f32: norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.09, avg = 0.01, max = 47.92, var = 1.00 | |
| 1.22.411.495 I 01706: f32: attn_post_norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -6.11, avg = 0.01, max = 59.15, var = 0.76 | |
| 1.22.431.208 I 01707: f32: attn_out-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.74, avg = 0.05, max = 303.09, var = 9.55 | |
| 1.22.451.130 I 01708: f32: norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.90, avg = 0.02, max = 70.38, var = 1.00 | |
| 1.22.470.666 I 01709: f32: ffn_norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -44.21, avg = -0.00, max = 35.27, var = 0.16 | |
| 1.22.646.685 I 01710: f32: ffn_gate-41 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.74, avg = -0.65, max = 8.77, var = 0.51 | |
| 1.22.822.772 I 01711: f32: ffn_up-41 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.05, avg = -0.00, max = 29.64, var = 0.31 | |
| 1.22.903.767 I 01712: f32: ffn_geglu-41 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -223.18, avg = 0.00, max = 81.44, var = 0.03 | |
| 1.23.020.549 I 01713: f32: ffn_out-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.22, avg = 0.00, max = 21.66, var = 0.24 | |
| 1.23.040.446 I 01714: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.98, avg = 0.00, max = 49.32, var = 1.00 | |
| 1.23.060.179 I 01715: f32: ffn_post_norm-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.81, avg = 0.01, max = 94.78, var = 1.78 | |
| 1.23.080.046 I 01716: f32: node_1715 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -74.73, avg = 0.06, max = 396.95, var = 16.28 | |
| 1.23.100.030 I 01717: f32: l_out-41 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.33, avg = 0.04, max = 262.05, var = 7.10 | |
| 1.23.120.026 I 01718: f32: norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.47, avg = 0.02, max = 69.40, var = 1.00 | |
| 1.23.139.599 I 01719: f32: attn_norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -512.59, avg = -0.00, max = 44.93, var = 0.55 | |
| 1.23.207.220 I 01720: f32: Qcur-42 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.98, avg = 0.00, max = 52.19, var = 0.99 | |
| 1.23.237.421 I 01721: f32: Qcur-42 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.98, avg = 0.00, max = 52.19, var = 0.99 | |
| 1.23.268.479 I 01722: f32: norm-42 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.63, avg = -0.00, max = 13.10, var = 1.00 | |
| 1.23.298.133 I 01723: f32: Qcur_normed-42 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.36, avg = -0.00, max = 12.85, var = 0.96 | |
| 1.23.327.819 I 01724: f32: Qcur_pos-42 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.41, avg = -0.01, max = 13.22, var = 0.96 | |
| 1.23.362.073 I 01725: f32: Kcur-42 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -93.49, avg = 0.01, max = 103.66, var = 1.16 | |
| 1.23.377.479 I 01726: f32: Kcur-42 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -93.49, avg = 0.01, max = 103.66, var = 1.16 | |
| 1.23.393.079 I 01727: f32: norm-42 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.94, avg = 0.01, max = 12.16, var = 1.00 | |
| 1.23.408.375 I 01728: f32: Kcur_normed-42 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.52, avg = 0.00, max = 1.54, var = 0.02 | |
| 1.23.424.202 I 01729: f32: Kcur_pos-42 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.54, var = 0.02 | |
| 1.23.467.424 I 01730: f32: Vcur-42 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -370.13, avg = -0.00, max = 382.91, var = 1.94 | |
| 1.23.482.546 I 01731: f32: Vcur-42 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -370.13, avg = -0.00, max = 382.91, var = 1.94 | |
| 1.23.497.895 I 01732: f32: Vcur_normed-42 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.23.512.596 I 01733: f32: Kcur_pos-42 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.54, var = 0.02 | |
| 1.23.556.621 I 01734: f16: cache_k_l42 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.54, var = 0.01 | |
| 1.23.571.291 I 01735: f32: Vcur_normed-42 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.23.615.200 I 01736: f16: cache_v_l42 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 0.33 | |
| 1.23.644.478 I 01737: f32: Qcur_pos-42 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.41, avg = -0.01, max = 13.22, var = 0.96 | |
| 1.23.673.753 I 01738: f32: Qcur_pos-42 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.41, avg = -0.01, max = 13.22, var = 0.96 | |
| 1.23.688.352 I 01739: f16: cache_k_l42 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.54, var = 0.02 | |
| 1.23.702.964 I 01740: f16: cache_k_l42 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.51, avg = 0.00, max = 1.54, var = 0.02 | |
| 1.23.717.527 I 01741: f16: cache_v_l42 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.23.732.086 I 01742: f16: cache_v_l42 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.23.771.917 I 01743: f32: __fattn__-42 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.97, var = 0.24 | |
| 1.23.801.295 I 01744: f32: kqv_out-42 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.97, var = 0.24 | |
| 1.23.858.598 I 01745: f32: node_1744 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.29, avg = 0.00, max = 34.51, var = 0.71 | |
| 1.23.878.567 I 01746: f32: norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.37, avg = 0.00, max = 38.10, var = 1.00 | |
| 1.23.898.112 I 01747: f32: attn_post_norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.34, avg = 0.01, max = 33.49, var = 0.14 | |
| 1.23.917.950 I 01748: f32: attn_out-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.18, avg = 0.05, max = 283.00, var = 8.28 | |
| 1.23.937.877 I 01749: f32: norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.98, avg = 0.02, max = 68.67, var = 1.00 | |
| 1.23.957.386 I 01750: f32: ffn_norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.51, avg = -0.00, max = 43.44, var = 0.21 | |
| 1.24.133.372 I 01751: f32: ffn_gate-42 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.14, avg = -0.83, max = 8.60, var = 0.62 | |
| 1.24.309.056 I 01752: f32: ffn_up-42 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -72.95, avg = 0.00, max = 59.82, var = 0.45 | |
| 1.24.390.126 I 01753: f32: ffn_geglu-42 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -154.44, avg = -0.00, max = 243.41, var = 0.04 | |
| 1.24.506.748 I 01754: f32: ffn_out-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.54, avg = 0.00, max = 24.25, var = 0.52 | |
| 1.24.526.660 I 01755: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.91, avg = 0.01, max = 34.38, var = 1.00 | |
| 1.24.546.257 I 01756: f32: ffn_post_norm-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.20, avg = 0.01, max = 49.41, var = 0.64 | |
| 1.24.565.970 I 01757: f32: node_1756 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.95, avg = 0.06, max = 332.40, var = 11.55 | |
| 1.24.585.947 I 01758: f32: l_out-42 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.47, avg = 0.04, max = 254.50, var = 6.77 | |
| 1.24.605.872 I 01759: f32: norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.20, avg = 0.02, max = 67.96, var = 1.00 | |
| 1.24.625.398 I 01760: f32: attn_norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -273.29, avg = -0.00, max = 30.00, var = 0.23 | |
| 1.24.693.246 I 01761: f32: Qcur-43 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.20, avg = -0.01, max = 11.20, var = 0.38 | |
| 1.24.723.457 I 01762: f32: Qcur-43 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.20, avg = -0.01, max = 11.20, var = 0.38 | |
| 1.24.753.768 I 01763: f32: norm-43 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.59, avg = -0.01, max = 13.57, var = 1.00 | |
| 1.24.783.543 I 01764: f32: Qcur_normed-43 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.32, avg = -0.01, max = 13.31, var = 0.96 | |
| 1.24.813.475 I 01765: f32: Qcur_pos-43 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.99, avg = -0.02, max = 13.29, var = 0.96 | |
| 1.24.847.609 I 01766: f32: Kcur-43 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -60.36, avg = 0.01, max = 54.58, var = 0.47 | |
| 1.24.862.730 I 01767: f32: Kcur-43 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -60.36, avg = 0.01, max = 54.58, var = 0.47 | |
| 1.24.878.275 I 01768: f32: norm-43 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.87, avg = 0.01, max = 13.44, var = 1.00 | |
| 1.24.893.256 I 01769: f32: Kcur_normed-43 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.65, avg = 0.00, max = 1.72, var = 0.02 | |
| 1.24.908.426 I 01770: f32: Kcur_pos-43 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.72, var = 0.02 | |
| 1.24.942.486 I 01771: f32: Vcur-43 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -356.91, avg = -0.00, max = 378.28, var = 1.73 | |
| 1.24.957.604 I 01772: f32: Vcur-43 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -356.91, avg = -0.00, max = 378.28, var = 1.73 | |
| 1.24.972.908 I 01773: f32: Vcur_normed-43 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 1.00 | |
| 1.24.987.595 I 01774: f32: Kcur_pos-43 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.72, var = 0.02 | |
| 1.25.031.670 I 01775: f16: cache_k_l43 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.72, var = 0.01 | |
| 1.25.046.330 I 01776: f32: Vcur_normed-43 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 1.00 | |
| 1.25.090.246 I 01777: f16: cache_v_l43 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 0.33 | |
| 1.25.119.533 I 01778: f32: Qcur_pos-43 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.99, avg = -0.02, max = 13.29, var = 0.96 | |
| 1.25.148.802 I 01779: f32: Qcur_pos-43 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.99, avg = -0.02, max = 13.29, var = 0.96 | |
| 1.25.163.406 I 01780: f16: cache_k_l43 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.72, var = 0.02 | |
| 1.25.178.011 I 01781: f16: cache_k_l43 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.63, avg = 0.00, max = 1.72, var = 0.02 | |
| 1.25.192.562 I 01782: f16: cache_v_l43 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 1.00 | |
| 1.25.207.110 I 01783: f16: cache_v_l43 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 1.00 | |
| 1.25.247.056 I 01784: f32: __fattn__-43 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 0.25 | |
| 1.25.276.493 I 01785: f32: kqv_out-43 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 0.25 | |
| 1.25.333.642 I 01786: f32: node_1785 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.50, avg = -0.01, max = 14.87, var = 0.62 | |
| 1.25.353.492 I 01787: f32: norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.59, avg = -0.01, max = 18.12, var = 1.00 | |
| 1.25.373.057 I 01788: f32: attn_post_norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.17, avg = -0.01, max = 14.79, var = 0.13 | |
| 1.25.392.870 I 01789: f32: attn_out-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.33, avg = 0.04, max = 259.11, var = 6.97 | |
| 1.25.412.835 I 01790: f32: norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.87, avg = 0.01, max = 67.64, var = 1.00 | |
| 1.25.432.346 I 01791: f32: ffn_norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.49, avg = -0.00, max = 42.73, var = 0.33 | |
| 1.25.608.259 I 01792: f32: ffn_gate-43 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.48, avg = -1.44, max = 10.34, var = 0.82 | |
| 1.25.784.873 I 01793: f32: ffn_up-43 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -42.11, avg = -0.00, max = 46.10, var = 0.64 | |
| 1.25.867.472 I 01794: f32: ffn_geglu-43 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -248.24, avg = 0.00, max = 42.69, var = 0.07 | |
| 1.25.985.571 I 01795: f32: ffn_out-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.49, avg = -0.02, max = 24.15, var = 0.68 | |
| 1.26.006.584 I 01796: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.28, avg = -0.02, max = 18.68, var = 1.00 | |
| 1.26.027.136 I 01797: f32: ffn_post_norm-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.41, avg = -0.00, max = 22.32, var = 0.28 | |
| 1.26.047.235 I 01798: f32: node_1797 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.71, avg = 0.03, max = 271.76, var = 7.86 | |
| 1.26.068.149 I 01799: f32: l_out-43 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.51, avg = 0.03, max = 220.80, var = 5.19 | |
| 1.26.088.680 I 01800: f32: norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.98, avg = 0.01, max = 66.75, var = 1.00 | |
| 1.26.108.515 I 01801: f32: attn_norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -536.59, avg = -0.00, max = 58.19, var = 0.81 | |
| 1.26.179.978 I 01802: f32: Qcur-44 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.13, avg = 0.00, max = 31.77, var = 1.66 | |
| 1.26.210.542 I 01803: f32: Qcur-44 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.13, avg = 0.00, max = 31.77, var = 1.66 | |
| 1.26.241.351 I 01804: f32: norm-44 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.49, avg = 0.00, max = 14.56, var = 1.00 | |
| 1.26.270.952 I 01805: f32: Qcur_normed-44 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.33, avg = 0.00, max = 14.39, var = 0.98 | |
| 1.26.300.921 I 01806: f32: Qcur_pos-44 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.30, avg = -0.00, max = 13.97, var = 0.98 | |
| 1.26.334.916 I 01807: f32: Kcur-44 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -127.25, avg = 0.01, max = 98.82, var = 1.93 | |
| 1.26.350.019 I 01808: f32: Kcur-44 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -127.25, avg = 0.01, max = 98.82, var = 1.93 | |
| 1.26.365.351 I 01809: f32: norm-44 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.40, avg = 0.01, max = 12.59, var = 1.00 | |
| 1.26.380.374 I 01810: f32: Kcur_normed-44 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.70, avg = 0.00, max = 1.60, var = 0.02 | |
| 1.26.395.309 I 01811: f32: Kcur_pos-44 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.69, avg = 0.00, max = 1.59, var = 0.02 | |
| 1.26.429.459 I 01812: f32: Vcur-44 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -622.38, avg = 0.01, max = 694.70, var = 6.09 | |
| 1.26.444.556 I 01813: f32: Vcur-44 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -622.38, avg = 0.01, max = 694.70, var = 6.09 | |
| 1.26.459.820 I 01814: f32: Vcur_normed-44 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.26.474.539 I 01815: f32: Kcur_pos-44 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.69, avg = 0.00, max = 1.59, var = 0.02 | |
| 1.26.518.596 I 01816: f16: cache_k_l44 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.69, avg = 0.00, max = 1.59, var = 0.01 | |
| 1.26.533.249 I 01817: f32: Vcur_normed-44 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.26.577.204 I 01818: f16: cache_v_l44 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 0.33 | |
| 1.26.606.499 I 01819: f32: Qcur_pos-44 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.30, avg = -0.00, max = 13.97, var = 0.98 | |
| 1.26.635.772 I 01820: f32: Qcur_pos-44 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.30, avg = -0.00, max = 13.97, var = 0.98 | |
| 1.26.650.379 I 01821: f16: cache_k_l44 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.69, avg = 0.00, max = 1.59, var = 0.02 | |
| 1.26.664.982 I 01822: f16: cache_k_l44 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.69, avg = 0.00, max = 1.59, var = 0.02 | |
| 1.26.679.531 I 01823: f16: cache_v_l44 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.26.694.079 I 01824: f16: cache_v_l44 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.26.733.944 I 01825: f32: __fattn__-44 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 0.23 | |
| 1.26.763.357 I 01826: f32: kqv_out-44 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.98, var = 0.23 | |
| 1.26.820.638 I 01827: f32: node_1826 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.59, avg = -0.01, max = 26.57, var = 0.69 | |
| 1.26.840.527 I 01828: f32: norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.41, avg = -0.01, max = 23.90, var = 1.00 | |
| 1.26.860.114 I 01829: f32: attn_post_norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.55, avg = -0.00, max = 9.60, var = 0.06 | |
| 1.26.879.842 I 01830: f32: attn_out-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.52, avg = 0.03, max = 225.66, var = 5.56 | |
| 1.26.899.788 I 01831: f32: norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.36, avg = 0.01, max = 66.14, var = 1.00 | |
| 1.26.919.392 I 01832: f32: ffn_norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.76, avg = -0.01, max = 46.02, var = 0.55 | |
| 1.27.095.145 I 01833: f32: ffn_gate-44 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.30, avg = -1.99, max = 9.85, var = 0.94 | |
| 1.27.270.671 I 01834: f32: ffn_up-44 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.75, avg = -0.00, max = 13.42, var = 0.77 | |
| 1.27.351.916 I 01835: f32: ffn_geglu-44 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.85, avg = 0.00, max = 64.65, var = 0.06 | |
| 1.27.468.667 I 01836: f32: ffn_out-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.63, avg = -0.01, max = 16.98, var = 0.62 | |
| 1.27.488.601 I 01837: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.75, avg = -0.01, max = 14.44, var = 1.00 | |
| 1.27.508.224 I 01838: f32: ffn_post_norm-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.68, avg = 0.00, max = 15.56, var = 0.22 | |
| 1.27.528.018 I 01839: f32: node_1838 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -81.89, avg = 0.03, max = 241.23, var = 6.58 | |
| 1.27.548.063 I 01840: f32: l_out-44 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -69.10, avg = 0.02, max = 203.54, var = 4.68 | |
| 1.27.568.014 I 01841: f32: norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.18, avg = 0.01, max = 64.63, var = 1.00 | |
| 1.27.587.576 I 01842: f32: attn_norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -439.62, avg = -0.00, max = 49.36, var = 0.60 | |
| 1.27.655.232 I 01843: f32: Qcur-45 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.92, avg = -0.02, max = 29.89, var = 1.18 | |
| 1.27.685.432 I 01844: f32: Qcur-45 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.92, avg = -0.02, max = 29.89, var = 1.18 | |
| 1.27.715.783 I 01845: f32: norm-45 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.96, avg = -0.03, max = 13.61, var = 1.00 | |
| 1.27.745.424 I 01846: f32: Qcur_normed-45 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.39, avg = -0.03, max = 14.03, var = 1.06 | |
| 1.27.775.142 I 01847: f32: Qcur_pos-45 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.29, avg = -0.02, max = 13.95, var = 1.06 | |
| 1.27.809.297 I 01848: f32: Kcur-45 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -81.96, avg = 0.00, max = 85.48, var = 1.59 | |
| 1.27.824.112 I 01849: f32: Kcur-45 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -81.96, avg = 0.00, max = 85.48, var = 1.59 | |
| 1.27.839.400 I 01850: f32: norm-45 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.13, avg = -0.00, max = 14.52, var = 1.00 | |
| 1.27.854.374 I 01851: f32: Kcur_normed-45 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.59, avg = -0.00, max = 1.76, var = 0.01 | |
| 1.27.869.531 I 01852: f32: Kcur_pos-45 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.58, avg = 0.00, max = 1.75, var = 0.01 | |
| 1.27.903.516 I 01853: f32: Vcur-45 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -475.60, avg = 0.00, max = 532.00, var = 3.75 | |
| 1.27.918.622 I 01854: f32: Vcur-45 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -475.60, avg = 0.00, max = 532.00, var = 3.75 | |
| 1.27.933.931 I 01855: f32: Vcur_normed-45 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 1.27.948.628 I 01856: f32: Kcur_pos-45 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.58, avg = 0.00, max = 1.75, var = 0.01 | |
| 1.27.992.717 I 01857: f16: cache_k_l45 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.58, avg = 0.00, max = 1.75, var = 0.00 | |
| 1.28.007.370 I 01858: f32: Vcur_normed-45 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 1.28.051.306 I 01859: f16: cache_v_l45 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 0.33 | |
| 1.28.080.580 I 01860: f32: Qcur_pos-45 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.29, avg = -0.02, max = 13.95, var = 1.06 | |
| 1.28.109.853 I 01861: f32: Qcur_pos-45 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.29, avg = -0.02, max = 13.95, var = 1.06 | |
| 1.28.124.474 I 01862: f16: cache_k_l45 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.58, avg = 0.00, max = 1.75, var = 0.01 | |
| 1.28.139.125 I 01863: f16: cache_k_l45 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.58, avg = 0.00, max = 1.75, var = 0.01 | |
| 1.28.153.673 I 01864: f16: cache_v_l45 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 1.28.168.218 I 01865: f16: cache_v_l45 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 1.00 | |
| 1.28.208.055 I 01866: f32: __fattn__-45 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.98, var = 0.21 | |
| 1.28.237.466 I 01867: f32: kqv_out-45 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.98, var = 0.21 | |
| 1.28.295.222 I 01868: f32: node_1867 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.77, avg = 0.00, max = 20.82, var = 0.61 | |
| 1.28.315.186 I 01869: f32: norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.07, avg = 0.00, max = 19.24, var = 1.00 | |
| 1.28.334.735 I 01870: f32: attn_post_norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.62, avg = -0.00, max = 6.67, var = 0.06 | |
| 1.28.354.442 I 01871: f32: attn_out-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -76.48, avg = 0.02, max = 204.09, var = 4.88 | |
| 1.28.374.352 I 01872: f32: norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.26, avg = 0.01, max = 63.91, var = 1.00 | |
| 1.28.393.853 I 01873: f32: ffn_norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -34.46, avg = -0.01, max = 40.17, var = 0.45 | |
| 1.28.570.271 I 01874: f32: ffn_gate-45 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.89, avg = -1.67, max = 12.83, var = 0.93 | |
| 1.28.745.321 I 01875: f32: ffn_up-45 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.35, avg = -0.00, max = 10.46, var = 0.68 | |
| 1.28.826.436 I 01876: f32: ffn_geglu-45 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -73.85, avg = -0.00, max = 86.62, var = 0.09 | |
| 1.28.943.246 I 01877: f32: ffn_out-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.61, avg = -0.00, max = 16.60, var = 0.73 | |
| 1.28.963.197 I 01878: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.99, avg = -0.00, max = 15.83, var = 1.00 | |
| 1.28.982.807 I 01879: f32: ffn_post_norm-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.36, avg = -0.00, max = 8.36, var = 0.17 | |
| 1.29.002.517 I 01880: f32: node_1879 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -92.30, avg = 0.02, max = 208.35, var = 5.43 | |
| 1.29.022.511 I 01881: f32: l_out-45 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -80.04, avg = 0.02, max = 180.68, var = 4.09 | |
| 1.29.042.453 I 01882: f32: norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.10, avg = 0.01, max = 62.97, var = 1.00 | |
| 1.29.061.979 I 01883: f32: attn_norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -415.01, avg = -0.01, max = 43.34, var = 0.56 | |
| 1.29.129.565 I 01884: f32: Qcur-46 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.18, avg = -0.00, max = 30.60, var = 1.23 | |
| 1.29.158.926 I 01885: f32: Qcur-46 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.18, avg = -0.00, max = 30.60, var = 1.23 | |
| 1.29.189.220 I 01886: f32: norm-46 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.00, avg = -0.00, max = 13.43, var = 1.00 | |
| 1.29.218.952 I 01887: f32: Qcur_normed-46 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.22, avg = -0.00, max = 13.64, var = 1.03 | |
| 1.29.248.942 I 01888: f32: Qcur_pos-46 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.00, max = 13.94, var = 1.03 | |
| 1.29.283.136 I 01889: f32: Kcur-46 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -75.76, avg = 0.02, max = 77.25, var = 1.30 | |
| 1.29.298.242 I 01890: f32: Kcur-46 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -75.76, avg = 0.02, max = 77.25, var = 1.30 | |
| 1.29.313.611 I 01891: f32: norm-46 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.80, avg = 0.02, max = 12.33, var = 1.00 | |
| 1.29.328.580 I 01892: f32: Kcur_normed-46 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.70, avg = 0.00, max = 1.52, var = 0.02 | |
| 1.29.343.747 I 01893: f32: Kcur_pos-46 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.68, avg = 0.00, max = 1.50, var = 0.02 | |
| 1.29.377.984 I 01894: f32: Vcur-46 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -334.67, avg = -0.01, max = 398.60, var = 2.18 | |
| 1.29.393.097 I 01895: f32: Vcur-46 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -334.67, avg = -0.01, max = 398.60, var = 2.18 | |
| 1.29.408.406 I 01896: f32: Vcur_normed-46 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 16.00, var = 1.00 | |
| 1.29.423.104 I 01897: f32: Kcur_pos-46 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.68, avg = 0.00, max = 1.50, var = 0.02 | |
| 1.29.467.163 I 01898: f16: cache_k_l46 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.68, avg = 0.00, max = 1.50, var = 0.01 | |
| 1.29.481.819 I 01899: f32: Vcur_normed-46 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 16.00, var = 1.00 | |
| 1.29.525.780 I 01900: f16: cache_v_l46 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 16.00, var = 0.33 | |
| 1.29.555.045 I 01901: f32: Qcur_pos-46 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.00, max = 13.94, var = 1.03 | |
| 1.29.584.329 I 01902: f32: Qcur_pos-46 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.00, max = 13.94, var = 1.03 | |
| 1.29.598.939 I 01903: f16: cache_k_l46 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.68, avg = 0.00, max = 1.50, var = 0.02 | |
| 1.29.613.541 I 01904: f16: cache_k_l46 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.68, avg = 0.00, max = 1.50, var = 0.02 | |
| 1.29.628.078 I 01905: f16: cache_v_l46 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 16.00, var = 1.00 | |
| 1.29.642.616 I 01906: f16: cache_v_l46 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 16.00, var = 1.00 | |
| 1.29.682.619 I 01907: f32: __fattn__-46 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.98, var = 0.23 | |
| 1.29.712.023 I 01908: f32: kqv_out-46 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.98, var = 0.23 | |
| 1.29.769.676 I 01909: f32: node_1908 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.73, avg = -0.00, max = 11.38, var = 0.44 | |
| 1.29.789.609 I 01910: f32: norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.06, avg = -0.00, max = 15.39, var = 1.00 | |
| 1.29.809.310 I 01911: f32: attn_post_norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.96, avg = -0.00, max = 5.52, var = 0.05 | |
| 1.29.829.058 I 01912: f32: attn_out-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -83.59, avg = 0.01, max = 178.91, var = 4.17 | |
| 1.29.848.987 I 01913: f32: norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.71, avg = 0.01, max = 62.08, var = 1.00 | |
| 1.29.868.521 I 01914: f32: ffn_norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.28, avg = -0.01, max = 47.72, var = 0.55 | |
| 1.30.044.712 I 01915: f32: ffn_gate-46 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -8.08, avg = -1.83, max = 11.34, var = 0.96 | |
| 1.30.220.565 I 01916: f32: ffn_up-46 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.63, avg = 0.01, max = 51.50, var = 0.72 | |
| 1.30.301.665 I 01917: f32: ffn_geglu-46 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -137.10, avg = 0.00, max = 62.72, var = 0.09 | |
| 1.30.418.165 I 01918: f32: ffn_out-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.03, avg = 0.01, max = 19.20, var = 0.76 | |
| 1.30.438.139 I 01919: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.67, avg = 0.01, max = 15.39, var = 1.00 | |
| 1.30.457.746 I 01920: f32: ffn_post_norm-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.73, avg = 0.00, max = 9.88, var = 0.15 | |
| 1.30.477.477 I 01921: f32: node_1920 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -102.11, avg = 0.02, max = 188.78, var = 4.79 | |
| 1.30.497.418 I 01922: f32: l_out-46 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -90.94, avg = 0.01, max = 168.13, var = 3.80 | |
| 1.30.517.361 I 01923: f32: norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.73, avg = 0.01, max = 60.63, var = 1.00 | |
| 1.30.536.926 I 01924: f32: attn_norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -187.32, avg = -0.00, max = 32.64, var = 0.15 | |
| 1.30.671.403 I 01925: f32: Qcur-47 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.19, avg = -0.00, max = 18.54, var = 0.21 | |
| 1.30.730.115 I 01926: f32: Qcur-47 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.19, avg = -0.00, max = 18.54, var = 0.21 | |
| 1.30.791.035 I 01927: f32: norm-47 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.11, avg = -0.01, max = 16.04, var = 1.00 | |
| 1.30.850.309 I 01928: f32: Qcur_normed-47 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.82, avg = -0.01, max = 15.79, var = 0.97 | |
| 1.30.909.745 I 01929: f32: Qcur_pos-47 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.37, avg = -0.01, max = 19.17, var = 0.97 | |
| 1.30.927.758 I 01930: f32: Vcur-47 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.39, avg = -0.00, max = 53.21, var = 0.22 | |
| 1.30.935.220 I 01931: f32: Vcur-47 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.39, avg = -0.00, max = 53.21, var = 0.22 | |
| 1.30.943.161 I 01932: f32: norm-47 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.51, avg = -0.01, max = 20.26, var = 1.00 | |
| 1.30.950.832 I 01933: f32: Kcur_normed-47 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.29, var = 0.00 | |
| 1.30.958.614 I 01934: f32: Kcur_pos-47 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.29, var = 0.00 | |
| 1.30.966.081 I 01935: f32: Vcur-47 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -59.39, avg = -0.00, max = 53.21, var = 0.22 | |
| 1.30.974.025 I 01936: f32: Vcur_normed-47 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.51, avg = -0.01, max = 20.26, var = 1.00 | |
| 1.30.981.493 I 01937: f32: Kcur_pos-47 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.29, var = 0.00 | |
| 1.31.040.621 I 01938: f16: cache_k_l47 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.29, var = 0.00 | |
| 1.31.048.031 I 01939: f32: Vcur_normed-47 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.51, avg = -0.01, max = 20.26, var = 1.00 | |
| 1.31.106.962 I 01940: f16: cache_v_l47 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.52, avg = -0.00, max = 20.27, var = 0.12 | |
| 1.31.165.707 I 01941: f32: Qcur_pos-47 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.37, avg = -0.01, max = 19.17, var = 0.97 | |
| 1.31.224.439 I 01942: f32: Qcur_pos-47 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.37, avg = -0.01, max = 19.17, var = 0.97 | |
| 1.31.231.778 I 01943: f16: cache_k_l47 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.29, var = 0.00 | |
| 1.31.239.126 I 01944: f16: cache_k_l47 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.43, avg = -0.00, max = 1.29, var = 0.00 | |
| 1.31.246.416 I 01945: f16: cache_v_l47 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.52, avg = -0.01, max = 20.27, var = 1.00 | |
| 1.31.253.715 I 01946: f16: cache_v_l47 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.52, avg = -0.01, max = 20.27, var = 1.00 | |
| 1.31.331.894 I 01947: f32: __fattn__-47 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.70, avg = 0.00, max = 20.08, var = 0.30 | |
| 1.31.390.686 I 01948: f32: kqv_out-47 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.70, avg = 0.00, max = 20.08, var = 0.30 | |
| 1.31.484.846 I 01949: f32: node_1948 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.47, avg = -0.01, max = 30.36, var = 1.39 | |
| 1.31.504.724 I 01950: f32: norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.31, avg = -0.01, max = 19.36, var = 1.00 | |
| 1.31.524.285 I 01951: f32: attn_post_norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.81, avg = -0.00, max = 6.74, var = 0.09 | |
| 1.31.543.989 I 01952: f32: attn_out-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -100.52, avg = 0.01, max = 168.90, var = 4.19 | |
| 1.31.563.933 I 01953: f32: norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.50, avg = 0.01, max = 58.90, var = 1.00 | |
| 1.31.583.467 I 01954: f32: ffn_norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.90, avg = -0.00, max = 46.73, var = 0.51 | |
| 1.31.759.369 I 01955: f32: ffn_gate-47 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.16, avg = -1.70, max = 12.31, var = 0.94 | |
| 1.31.934.803 I 01956: f32: ffn_up-47 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -65.01, avg = -0.00, max = 64.82, var = 0.74 | |
| 1.32.016.148 I 01957: f32: ffn_geglu-47 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -183.60, avg = 0.00, max = 232.05, var = 0.11 | |
| 1.32.132.859 I 01958: f32: ffn_out-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.92, avg = -0.01, max = 20.16, var = 0.78 | |
| 1.32.152.806 I 01959: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.21, avg = -0.02, max = 14.87, var = 1.00 | |
| 1.32.172.463 I 01960: f32: ffn_post_norm-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.43, avg = -0.00, max = 9.63, var = 0.16 | |
| 1.32.192.173 I 01961: f32: node_1960 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -116.96, avg = 0.01, max = 178.53, var = 4.97 | |
| 1.32.211.984 I 01962: f32: l_out-47 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -102.79, avg = 0.01, max = 156.91, var = 3.84 | |
| 1.32.231.898 I 01963: f32: norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -40.35, avg = 0.00, max = 57.22, var = 1.00 | |
| 1.32.251.654 I 01964: f32: attn_norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -299.86, avg = -0.00, max = 36.93, var = 0.46 | |
| 1.32.319.165 I 01965: f32: Qcur-48 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.19, avg = 0.01, max = 26.79, var = 0.90 | |
| 1.32.349.374 I 01966: f32: Qcur-48 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.19, avg = 0.01, max = 26.79, var = 0.90 | |
| 1.32.379.655 I 01967: f32: norm-48 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.86, avg = 0.01, max = 13.48, var = 1.00 | |
| 1.32.409.457 I 01968: f32: Qcur_normed-48 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.18, avg = 0.01, max = 13.79, var = 1.05 | |
| 1.32.439.486 I 01969: f32: Qcur_pos-48 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.01, max = 13.98, var = 1.05 | |
| 1.32.473.403 I 01970: f32: Kcur-48 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -56.59, avg = 0.01, max = 68.09, var = 1.41 | |
| 1.32.488.510 I 01971: f32: Kcur-48 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -56.59, avg = 0.01, max = 68.09, var = 1.41 | |
| 1.32.503.857 I 01972: f32: norm-48 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.57, avg = 0.01, max = 11.45, var = 1.00 | |
| 1.32.518.818 I 01973: f32: Kcur_normed-48 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = 0.00, max = 1.40, var = 0.01 | |
| 1.32.533.954 I 01974: f32: Kcur_pos-48 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = 0.00, max = 1.57, var = 0.01 | |
| 1.32.568.452 I 01975: f32: Vcur-48 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -509.43, avg = 0.01, max = 468.09, var = 4.27 | |
| 1.32.583.250 I 01976: f32: Vcur-48 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -509.43, avg = 0.01, max = 468.09, var = 4.27 | |
| 1.32.598.361 I 01977: f32: Vcur_normed-48 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.32.613.064 I 01978: f32: Kcur_pos-48 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = 0.00, max = 1.57, var = 0.01 | |
| 1.32.657.160 I 01979: f16: cache_k_l48 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = 0.00, max = 1.57, var = 0.00 | |
| 1.32.671.840 I 01980: f32: Vcur_normed-48 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.32.715.805 I 01981: f16: cache_v_l48 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 15.99, var = 0.33 | |
| 1.32.745.144 I 01982: f32: Qcur_pos-48 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.01, max = 13.98, var = 1.05 | |
| 1.32.774.439 I 01983: f32: Qcur_pos-48 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.85, avg = 0.01, max = 13.98, var = 1.05 | |
| 1.32.789.061 I 01984: f16: cache_k_l48 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = 0.00, max = 1.57, var = 0.01 | |
| 1.32.803.694 I 01985: f16: cache_k_l48 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = 0.00, max = 1.57, var = 0.01 | |
| 1.32.818.273 I 01986: f16: cache_v_l48 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.32.832.816 I 01987: f16: cache_v_l48 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.32.872.505 I 01988: f32: __fattn__-48 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.99, var = 0.23 | |
| 1.32.901.947 I 01989: f32: kqv_out-48 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.01, max = 15.99, var = 0.23 | |
| 1.32.959.350 I 01990: f32: node_1989 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.95, avg = 0.01, max = 15.77, var = 0.62 | |
| 1.32.979.307 I 01991: f32: norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.96, avg = 0.02, max = 15.26, var = 1.00 | |
| 1.32.998.905 I 01992: f32: attn_post_norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.89, avg = -0.00, max = 6.84, var = 0.05 | |
| 1.33.018.766 I 01993: f32: attn_out-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -111.77, avg = 0.01, max = 158.29, var = 4.28 | |
| 1.33.038.745 I 01994: f32: norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -42.46, avg = 0.00, max = 55.61, var = 1.00 | |
| 1.33.058.279 I 01995: f32: ffn_norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.79, avg = 0.00, max = 41.20, var = 0.50 | |
| 1.33.234.289 I 01996: f32: ffn_gate-48 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.54, avg = -1.75, max = 41.41, var = 0.90 | |
| 1.33.410.364 I 01997: f32: ffn_up-48 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.88, avg = -0.00, max = 23.15, var = 0.72 | |
| 1.33.491.539 I 01998: f32: ffn_geglu-48 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -234.23, avg = 0.00, max = 440.39, var = 0.10 | |
| 1.33.608.078 I 01999: f32: ffn_out-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -77.20, avg = -0.00, max = 22.27, var = 0.74 | |
| 1.33.628.154 I 02000: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -40.81, avg = -0.01, max = 13.36, var = 1.00 | |
| 1.33.647.922 I 02001: f32: ffn_post_norm-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -36.18, avg = -0.00, max = 10.57, var = 0.17 | |
| 1.33.667.634 I 02002: f32: node_2001 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -128.55, avg = 0.01, max = 168.29, var = 5.23 | |
| 1.33.687.644 I 02003: f32: l_out-48 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -112.48, avg = 0.01, max = 147.26, var = 4.01 | |
| 1.33.707.584 I 02004: f32: norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -44.52, avg = 0.00, max = 54.17, var = 1.00 | |
| 1.33.727.176 I 02005: f32: attn_norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -210.77, avg = -0.01, max = 37.61, var = 0.27 | |
| 1.33.795.150 I 02006: f32: Qcur-49 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.11, avg = -0.00, max = 24.54, var = 0.64 | |
| 1.33.825.363 I 02007: f32: Qcur-49 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.11, avg = -0.00, max = 24.54, var = 0.64 | |
| 1.33.855.674 I 02008: f32: norm-49 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.58, avg = -0.00, max = 14.73, var = 1.00 | |
| 1.33.885.351 I 02009: f32: Qcur_normed-49 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.47, avg = -0.00, max = 14.61, var = 0.98 | |
| 1.33.915.335 I 02010: f32: Qcur_pos-49 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.21, avg = -0.00, max = 14.62, var = 0.98 | |
| 1.33.949.397 I 02011: f32: Kcur-49 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.56, avg = -0.01, max = 52.34, var = 0.59 | |
| 1.33.964.510 I 02012: f32: Kcur-49 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.56, avg = -0.01, max = 52.34, var = 0.59 | |
| 1.33.980.062 I 02013: f32: norm-49 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.41, avg = -0.01, max = 14.04, var = 1.00 | |
| 1.33.995.055 I 02014: f32: Kcur_normed-49 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.56, avg = -0.00, max = 1.77, var = 0.02 | |
| 1.34.010.226 I 02015: f32: Kcur_pos-49 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.77, var = 0.02 | |
| 1.34.044.367 I 02016: f32: Vcur-49 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -256.29, avg = 0.00, max = 267.87, var = 1.31 | |
| 1.34.059.481 I 02017: f32: Vcur-49 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -256.29, avg = 0.00, max = 267.87, var = 1.31 | |
| 1.34.075.101 I 02018: f32: Vcur_normed-49 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.34.090.039 I 02019: f32: Kcur_pos-49 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.77, var = 0.02 | |
| 1.34.134.088 I 02020: f16: cache_k_l49 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.77, var = 0.01 | |
| 1.34.149.627 I 02021: f32: Vcur_normed-49 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.34.194.898 I 02022: f16: cache_v_l49 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.98, var = 0.33 | |
| 1.34.225.671 I 02023: f32: Qcur_pos-49 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.21, avg = -0.00, max = 14.62, var = 0.98 | |
| 1.34.255.871 I 02024: f32: Qcur_pos-49 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.21, avg = -0.00, max = 14.62, var = 0.98 | |
| 1.34.270.539 I 02025: f16: cache_k_l49 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.77, var = 0.02 | |
| 1.34.285.147 I 02026: f16: cache_k_l49 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = -0.00, max = 1.77, var = 0.02 | |
| 1.34.299.701 I 02027: f16: cache_v_l49 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.34.314.276 I 02028: f16: cache_v_l49 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = 0.00, max = 15.98, var = 1.00 | |
| 1.34.357.205 I 02029: f32: __fattn__-49 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 0.27 | |
| 1.34.387.470 I 02030: f32: kqv_out-49 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.97, avg = -0.00, max = 15.95, var = 0.27 | |
| 1.34.444.464 I 02031: f32: node_2030 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.99, avg = 0.01, max = 14.41, var = 0.74 | |
| 1.34.464.664 I 02032: f32: norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.18, avg = 0.01, max = 21.56, var = 1.00 | |
| 1.34.484.405 I 02033: f32: attn_post_norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.70, avg = -0.01, max = 16.67, var = 0.17 | |
| 1.34.504.106 I 02034: f32: attn_out-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -126.17, avg = 0.00, max = 143.92, var = 4.39 | |
| 1.34.524.004 I 02035: f32: norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -46.45, avg = 0.00, max = 53.45, var = 1.00 | |
| 1.34.543.549 I 02036: f32: ffn_norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.95, avg = 0.00, max = 55.40, var = 0.42 | |
| 1.34.719.534 I 02037: f32: ffn_gate-49 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.99, avg = -1.56, max = 16.33, var = 0.89 | |
| 1.34.894.261 I 02038: f32: ffn_up-49 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.65, avg = -0.00, max = 12.25, var = 0.76 | |
| 1.34.975.370 I 02039: f32: ffn_geglu-49 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -73.96, avg = -0.00, max = 49.04, var = 0.09 | |
| 1.35.091.935 I 02040: f32: ffn_out-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.66, avg = 0.00, max = 38.33, var = 0.87 | |
| 1.35.112.045 I 02041: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.40, avg = 0.00, max = 14.65, var = 1.00 | |
| 1.35.131.700 I 02042: f32: ffn_post_norm-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.24, avg = -0.00, max = 8.31, var = 0.15 | |
| 1.35.151.398 I 02043: f32: node_2042 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -133.15, avg = -0.00, max = 151.16, var = 4.88 | |
| 1.35.171.363 I 02044: f32: l_out-49 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -111.82, avg = -0.00, max = 126.95, var = 3.44 | |
| 1.35.191.296 I 02045: f32: norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -46.98, avg = -0.00, max = 53.24, var = 1.00 | |
| 1.35.210.823 I 02046: f32: attn_norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -269.50, avg = -0.00, max = 65.84, var = 0.94 | |
| 1.35.278.215 I 02047: f32: Qcur-50 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.42, avg = 0.00, max = 35.87, var = 1.86 | |
| 1.35.308.379 I 02048: f32: Qcur-50 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -48.42, avg = 0.00, max = 35.87, var = 1.86 | |
| 1.35.338.671 I 02049: f32: norm-50 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.53, avg = 0.00, max = 13.74, var = 1.00 | |
| 1.35.368.274 I 02050: f32: Qcur_normed-50 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.34, avg = 0.00, max = 13.53, var = 0.97 | |
| 1.35.398.219 I 02051: f32: Qcur_pos-50 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.11, avg = -0.01, max = 12.50, var = 0.97 | |
| 1.35.432.154 I 02052: f32: Kcur-50 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -140.66, avg = -0.01, max = 145.61, var = 2.98 | |
| 1.35.447.260 I 02053: f32: Kcur-50 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -140.66, avg = -0.01, max = 145.61, var = 2.98 | |
| 1.35.462.584 I 02054: f32: norm-50 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.98, avg = -0.01, max = 14.39, var = 1.00 | |
| 1.35.477.572 I 02055: f32: Kcur_normed-50 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.90, avg = -0.00, max = 1.83, var = 0.02 | |
| 1.35.493.260 I 02056: f32: Kcur_pos-50 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.90, avg = 0.00, max = 1.83, var = 0.02 | |
| 1.35.527.229 I 02057: f32: Vcur-50 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -710.17, avg = -0.01, max = 717.58, var = 9.57 | |
| 1.35.542.906 I 02058: f32: Vcur-50 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -710.17, avg = -0.01, max = 717.58, var = 9.57 | |
| 1.35.558.353 I 02059: f32: Vcur_normed-50 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.35.573.487 I 02060: f32: Kcur_pos-50 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.90, avg = 0.00, max = 1.83, var = 0.02 | |
| 1.35.617.415 I 02061: f16: cache_k_l50 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.90, avg = 0.00, max = 1.83, var = 0.01 | |
| 1.35.632.385 I 02062: f32: Vcur_normed-50 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.35.676.163 I 02063: f16: cache_v_l50 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 0.33 | |
| 1.35.706.086 I 02064: f32: Qcur_pos-50 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.11, avg = -0.01, max = 12.50, var = 0.97 | |
| 1.35.736.282 I 02065: f32: Qcur_pos-50 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.11, avg = -0.01, max = 12.50, var = 0.97 | |
| 1.35.750.930 I 02066: f16: cache_k_l50 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.90, avg = 0.00, max = 1.83, var = 0.02 | |
| 1.35.765.533 I 02067: f16: cache_k_l50 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.90, avg = 0.00, max = 1.83, var = 0.02 | |
| 1.35.780.078 I 02068: f16: cache_v_l50 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.35.794.629 I 02069: f16: cache_v_l50 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.00, max = 15.99, var = 1.00 | |
| 1.35.837.712 I 02070: f32: __fattn__-50 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.98, var = 0.28 | |
| 1.35.867.067 I 02071: f32: kqv_out-50 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.98, var = 0.28 | |
| 1.35.924.212 I 02072: f32: node_2071 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.51, avg = 0.02, max = 31.69, var = 5.02 | |
| 1.35.944.637 I 02073: f32: norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.58, avg = 0.01, max = 9.66, var = 1.00 | |
| 1.35.965.402 I 02074: f32: attn_post_norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.34, avg = -0.00, max = 4.76, var = 0.04 | |
| 1.35.985.442 I 02075: f32: attn_out-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -116.40, avg = -0.00, max = 128.35, var = 3.62 | |
| 1.36.005.802 I 02076: f32: norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -47.99, avg = -0.00, max = 52.79, var = 1.00 | |
| 1.36.026.570 I 02077: f32: ffn_norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.09, avg = 0.00, max = 24.60, var = 0.41 | |
| 1.36.202.305 I 02078: f32: ffn_gate-50 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.50, avg = -1.46, max = 12.30, var = 0.82 | |
| 1.36.385.811 I 02079: f32: ffn_up-50 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -45.73, avg = -0.01, max = 24.55, var = 0.81 | |
| 1.36.469.014 I 02080: f32: ffn_geglu-50 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -88.63, avg = 0.00, max = 60.33, var = 0.07 | |
| 1.36.586.007 I 02081: f32: ffn_out-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.88, avg = 0.00, max = 50.68, var = 1.11 | |
| 1.36.606.714 I 02082: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.10, avg = 0.00, max = 13.22, var = 1.00 | |
| 1.36.626.550 I 02083: f32: ffn_post_norm-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -5.39, avg = 0.00, max = 9.09, var = 0.13 | |
| 1.36.647.215 I 02084: f32: node_2083 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -109.11, avg = -0.00, max = 126.88, var = 3.61 | |
| 1.36.668.582 I 02085: f32: l_out-50 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -94.62, avg = -0.00, max = 110.03, var = 2.72 | |
| 1.36.688.995 I 02086: f32: norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -47.49, avg = -0.00, max = 54.06, var = 1.00 | |
| 1.36.709.745 I 02087: f32: attn_norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -368.19, avg = -0.00, max = 55.16, var = 1.31 | |
| 1.36.777.646 I 02088: f32: Qcur-51 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.65, avg = 0.01, max = 37.25, var = 3.34 | |
| 1.36.807.855 I 02089: f32: Qcur-51 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.65, avg = 0.01, max = 37.25, var = 3.34 | |
| 1.36.839.126 I 02090: f32: norm-51 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.45, avg = 0.01, max = 14.12, var = 1.00 | |
| 1.36.869.678 I 02091: f32: Qcur_normed-51 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.34, avg = 0.01, max = 14.01, var = 0.98 | |
| 1.36.899.594 I 02092: f32: Qcur_pos-51 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.66, avg = -0.00, max = 13.74, var = 0.98 | |
| 1.36.947.238 I 02093: f32: Kcur-51 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -154.96, avg = -0.00, max = 142.20, var = 4.13 | |
| 1.36.962.343 I 02094: f32: Kcur-51 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -154.96, avg = -0.00, max = 142.20, var = 4.13 | |
| 1.36.978.226 I 02095: f32: norm-51 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.51, avg = -0.00, max = 13.99, var = 1.00 | |
| 1.36.993.280 I 02096: f32: Kcur_normed-51 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.83, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.37.008.416 I 02097: f32: Kcur_pos-51 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.82, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.37.042.348 I 02098: f32: Vcur-51 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -799.74, avg = -0.01, max = 874.14, var = 11.62 | |
| 1.37.057.456 I 02099: f32: Vcur-51 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -799.74, avg = -0.01, max = 874.14, var = 11.62 | |
| 1.37.072.755 I 02100: f32: Vcur_normed-51 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.01, max = 16.00, var = 1.00 | |
| 1.37.087.465 I 02101: f32: Kcur_pos-51 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.82, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.37.131.466 I 02102: f16: cache_k_l51 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.82, avg = -0.00, max = 1.76, var = 0.01 | |
| 1.37.146.118 I 02103: f32: Vcur_normed-51 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.01, max = 16.00, var = 1.00 | |
| 1.37.190.059 I 02104: f16: cache_v_l51 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.00, max = 16.00, var = 0.33 | |
| 1.37.219.420 I 02105: f32: Qcur_pos-51 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.66, avg = -0.00, max = 13.74, var = 0.98 | |
| 1.37.248.687 I 02106: f32: Qcur_pos-51 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.66, avg = -0.00, max = 13.74, var = 0.98 | |
| 1.37.263.293 I 02107: f16: cache_k_l51 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.82, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.37.277.916 I 02108: f16: cache_k_l51 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.82, avg = -0.00, max = 1.76, var = 0.02 | |
| 1.37.292.480 I 02109: f16: cache_v_l51 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.01, max = 16.00, var = 1.00 | |
| 1.37.307.035 I 02110: f16: cache_v_l51 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = -0.01, max = 16.00, var = 1.00 | |
| 1.37.346.710 I 02111: f32: __fattn__-51 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 16.00, var = 0.28 | |
| 1.37.376.152 I 02112: f32: kqv_out-51 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 16.00, var = 0.28 | |
| 1.37.433.422 I 02113: f32: node_2112 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.46, avg = -0.02, max = 11.47, var = 0.55 | |
| 1.37.453.541 I 02114: f32: norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.78, avg = -0.03, max = 13.89, var = 1.00 | |
| 1.37.473.290 I 02115: f32: attn_post_norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -4.15, avg = -0.00, max = 5.24, var = 0.04 | |
| 1.37.492.998 I 02116: f32: attn_out-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -93.92, avg = -0.00, max = 109.88, var = 2.68 | |
| 1.37.512.936 I 02117: f32: norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -47.32, avg = -0.00, max = 53.86, var = 1.00 | |
| 1.37.532.458 I 02118: f32: ffn_norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.86, avg = -0.00, max = 31.54, var = 0.54 | |
| 1.37.707.980 I 02119: f32: ffn_gate-51 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.61, avg = -1.96, max = 25.24, var = 0.99 | |
| 1.37.883.290 I 02120: f32: ffn_up-51 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.41, avg = 0.01, max = 31.08, var = 1.16 | |
| 1.37.964.544 I 02121: f32: ffn_geglu-51 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -138.02, avg = -0.00, max = 65.47, var = 0.10 | |
| 1.38.081.221 I 02122: f32: ffn_out-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -30.65, avg = 0.00, max = 36.51, var = 1.04 | |
| 1.38.101.141 I 02123: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.12, avg = 0.00, max = 26.35, var = 1.00 | |
| 1.38.120.583 I 02124: f32: ffn_post_norm-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.13, avg = 0.00, max = 23.98, var = 0.24 | |
| 1.38.140.278 I 02125: f32: node_2124 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -79.85, avg = 0.00, max = 100.56, var = 2.33 | |
| 1.38.160.277 I 02126: f32: l_out-51 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -70.18, avg = 0.00, max = 88.38, var = 1.80 | |
| 1.38.180.201 I 02127: f32: norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -42.34, avg = 0.00, max = 56.41, var = 1.00 | |
| 1.38.199.909 I 02128: f32: attn_norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -254.05, avg = -0.01, max = 71.89, var = 1.57 | |
| 1.38.267.292 I 02129: f32: Qcur-52 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.10, avg = -0.02, max = 51.38, var = 3.39 | |
| 1.38.296.804 I 02130: f32: Qcur-52 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -53.10, avg = -0.02, max = 51.38, var = 3.39 | |
| 1.38.327.113 I 02131: f32: norm-52 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.62, avg = -0.01, max = 13.91, var = 1.00 | |
| 1.38.356.765 I 02132: f32: Qcur_normed-52 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.05, avg = -0.01, max = 14.35, var = 1.06 | |
| 1.38.386.988 I 02133: f32: Qcur_pos-52 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.94, avg = -0.01, max = 13.40, var = 1.06 | |
| 1.38.420.785 I 02134: f32: Kcur-52 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -134.40, avg = 0.03, max = 137.44, var = 5.53 | |
| 1.38.435.888 I 02135: f32: Kcur-52 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -134.40, avg = 0.03, max = 137.44, var = 5.53 | |
| 1.38.451.259 I 02136: f32: norm-52 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.07, avg = 0.01, max = 13.76, var = 1.00 | |
| 1.38.466.218 I 02137: f32: Kcur_normed-52 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.70, avg = 0.00, max = 1.67, var = 0.01 | |
| 1.38.481.356 I 02138: f32: Kcur_pos-52 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = 0.00, max = 1.67, var = 0.01 | |
| 1.38.515.279 I 02139: f32: Vcur-52 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -866.85, avg = 0.01, max = 842.02, var = 13.08 | |
| 1.38.530.386 I 02140: f32: Vcur-52 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -866.85, avg = 0.01, max = 842.02, var = 13.08 | |
| 1.38.545.700 I 02141: f32: Vcur_normed-52 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 1.38.560.388 I 02142: f32: Kcur_pos-52 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = 0.00, max = 1.67, var = 0.01 | |
| 1.38.604.448 I 02143: f16: cache_k_l52 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = 0.00, max = 1.67, var = 0.00 | |
| 1.38.619.098 I 02144: f32: Vcur_normed-52 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 1.38.663.043 I 02145: f16: cache_v_l52 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 0.33 | |
| 1.38.692.338 I 02146: f32: Qcur_pos-52 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.94, avg = -0.01, max = 13.40, var = 1.06 | |
| 1.38.721.605 I 02147: f32: Qcur_pos-52 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.94, avg = -0.01, max = 13.40, var = 1.06 | |
| 1.38.736.218 I 02148: f16: cache_k_l52 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = 0.00, max = 1.67, var = 0.01 | |
| 1.38.750.837 I 02149: f16: cache_k_l52 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.71, avg = 0.00, max = 1.67, var = 0.01 | |
| 1.38.765.418 I 02150: f16: cache_v_l52 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 1.38.779.964 I 02151: f16: cache_v_l52 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.00, max = 16.00, var = 1.00 | |
| 1.38.819.809 I 02152: f32: __fattn__-52 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 0.27 | |
| 1.38.849.232 I 02153: f32: kqv_out-52 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 0.27 | |
| 1.38.906.507 I 02154: f32: node_2153 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.78, avg = -0.01, max = 21.31, var = 1.01 | |
| 1.38.926.442 I 02155: f32: norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.92, avg = -0.01, max = 23.62, var = 1.00 | |
| 1.38.945.808 I 02156: f32: attn_post_norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.30, avg = 0.00, max = 17.62, var = 0.07 | |
| 1.38.965.509 I 02157: f32: attn_out-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -57.77, avg = 0.00, max = 84.67, var = 1.54 | |
| 1.38.985.570 I 02158: f32: norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -39.49, avg = 0.00, max = 58.72, var = 1.00 | |
| 1.39.005.246 I 02159: f32: ffn_norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.78, avg = -0.00, max = 25.54, var = 0.37 | |
| 1.39.180.905 I 02160: f32: ffn_gate-52 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.07, avg = -1.22, max = 18.24, var = 0.93 | |
| 1.39.355.382 I 02161: f32: ffn_up-52 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.73, avg = -0.00, max = 13.98, var = 0.96 | |
| 1.39.436.551 I 02162: f32: ffn_geglu-52 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -149.33, avg = -0.00, max = 49.42, var = 0.16 | |
| 1.39.553.376 I 02163: f32: ffn_out-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.68, avg = 0.00, max = 48.68, var = 2.09 | |
| 1.39.573.251 I 02164: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.24, avg = 0.00, max = 32.65, var = 1.00 | |
| 1.39.592.858 I 02165: f32: ffn_post_norm-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.22, avg = 0.00, max = 42.09, var = 0.48 | |
| 1.39.612.574 I 02166: f32: node_2165 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.99, avg = 0.00, max = 70.14, var = 1.30 | |
| 1.39.632.526 I 02167: f32: l_out-52 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.02, avg = 0.00, max = 56.17, var = 0.84 | |
| 1.39.652.410 I 02168: f32: norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.36, avg = 0.00, max = 65.03, var = 1.00 | |
| 1.39.671.936 I 02169: f32: attn_norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.24, avg = -0.01, max = 67.57, var = 1.40 | |
| 1.39.806.568 I 02170: f32: Qcur-53 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -55.54, avg = -0.01, max = 38.64, var = 1.86 | |
| 1.39.866.600 I 02171: f32: Qcur-53 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -55.54, avg = -0.01, max = 38.64, var = 1.86 | |
| 1.39.927.564 I 02172: f32: norm-53 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.90, avg = -0.00, max = 17.31, var = 1.00 | |
| 1.39.986.704 I 02173: f32: Qcur_normed-53 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.74, avg = -0.00, max = 17.17, var = 0.98 | |
| 1.40.046.241 I 02174: f32: Qcur_pos-53 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.74, avg = -0.00, max = 18.33, var = 0.98 | |
| 1.40.063.521 I 02175: f32: Vcur-53 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -95.05, avg = 0.01, max = 94.98, var = 1.34 | |
| 1.40.071.070 I 02176: f32: Vcur-53 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -95.05, avg = 0.01, max = 94.98, var = 1.34 | |
| 1.40.079.126 I 02177: f32: norm-53 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.46, avg = 0.01, max = 22.46, var = 1.00 | |
| 1.40.086.581 I 02178: f32: Kcur_normed-53 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.41, var = 0.00 | |
| 1.40.094.157 I 02179: f32: Kcur_pos-53 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.41, var = 0.00 | |
| 1.40.101.504 I 02180: f32: Vcur-53 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -95.05, avg = 0.01, max = 94.98, var = 1.34 | |
| 1.40.109.206 I 02181: f32: Vcur_normed-53 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.46, avg = 0.01, max = 22.46, var = 1.00 | |
| 1.40.116.609 I 02182: f32: Kcur_pos-53 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.41, avg = 0.00, max = 1.41, var = 0.00 | |
| 1.40.174.993 I 02183: f16: cache_k_l53 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.42, var = 0.00 | |
| 1.40.182.337 I 02184: f32: Vcur_normed-53 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.46, avg = 0.01, max = 22.46, var = 1.00 | |
| 1.40.240.727 I 02185: f16: cache_v_l53 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.45, avg = 0.00, max = 22.45, var = 0.12 | |
| 1.40.299.244 I 02186: f32: Qcur_pos-53 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.74, avg = -0.00, max = 18.33, var = 0.98 | |
| 1.40.357.841 I 02187: f32: Qcur_pos-53 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.74, avg = -0.00, max = 18.33, var = 0.98 | |
| 1.40.365.157 I 02188: f16: cache_k_l53 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.42, var = 0.00 | |
| 1.40.372.499 I 02189: f16: cache_k_l53 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.42, avg = 0.00, max = 1.42, var = 0.00 | |
| 1.40.379.774 I 02190: f16: cache_v_l53 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.45, avg = 0.01, max = 22.45, var = 1.00 | |
| 1.40.387.079 I 02191: f16: cache_v_l53 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.45, avg = 0.01, max = 22.45, var = 1.00 | |
| 1.40.464.737 I 02192: f32: __fattn__-53 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.23, avg = 0.00, max = 22.41, var = 0.34 | |
| 1.40.523.331 I 02193: f32: kqv_out-53 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.23, avg = 0.00, max = 22.41, var = 0.34 | |
| 1.40.616.801 I 02194: f32: node_2193 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -35.60, avg = -0.00, max = 42.54, var = 2.07 | |
| 1.40.636.807 I 02195: f32: norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.62, avg = -0.00, max = 33.96, var = 1.00 | |
| 1.40.656.464 I 02196: f32: attn_post_norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.46, avg = 0.00, max = 18.50, var = 0.19 | |
| 1.40.676.164 I 02197: f32: attn_out-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.50, avg = 0.00, max = 74.63, var = 1.14 | |
| 1.40.696.053 I 02198: f32: norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.71, avg = 0.00, max = 66.72, var = 1.00 | |
| 1.40.715.604 I 02199: f32: ffn_norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.51, avg = -0.00, max = 17.92, var = 0.47 | |
| 1.40.891.089 I 02200: f32: ffn_gate-53 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.40, avg = -1.63, max = 36.02, var = 1.12 | |
| 1.41.067.123 I 02201: f32: ffn_up-53 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.59, avg = 0.00, max = 40.33, var = 1.10 | |
| 1.41.148.193 I 02202: f32: ffn_geglu-53 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -276.31, avg = -0.00, max = 60.35, var = 0.16 | |
| 1.41.264.995 I 02203: f32: ffn_out-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.03, avg = -0.00, max = 45.45, var = 1.45 | |
| 1.41.285.051 I 02204: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.36, avg = -0.00, max = 29.80, var = 1.00 | |
| 1.41.304.869 I 02205: f32: ffn_post_norm-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -7.76, avg = 0.00, max = 26.77, var = 0.20 | |
| 1.41.324.597 I 02206: f32: node_2205 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.27, avg = 0.00, max = 94.31, var = 1.59 | |
| 1.41.344.393 I 02207: f32: l_out-53 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.96, avg = 0.00, max = 79.94, var = 1.14 | |
| 1.41.364.370 I 02208: f32: norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.50, avg = 0.00, max = 68.29, var = 1.00 | |
| 1.41.383.961 I 02209: f32: attn_norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -28.30, avg = -0.00, max = 78.14, var = 0.46 | |
| 1.41.451.512 I 02210: f32: Qcur-54 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.62, avg = -0.01, max = 24.92, var = 1.30 | |
| 1.41.481.724 I 02211: f32: Qcur-54 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.62, avg = -0.01, max = 24.92, var = 1.30 | |
| 1.41.512.034 I 02212: f32: norm-54 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.18, avg = -0.01, max = 13.63, var = 1.00 | |
| 1.41.541.493 I 02213: f32: Qcur_normed-54 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.29, avg = -0.01, max = 13.74, var = 1.02 | |
| 1.41.571.474 I 02214: f32: Qcur_pos-54 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = 0.00, max = 13.69, var = 1.02 | |
| 1.41.606.392 I 02215: f32: Kcur-54 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -70.54, avg = -0.01, max = 66.40, var = 1.53 | |
| 1.41.621.503 I 02216: f32: Kcur-54 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -70.54, avg = -0.01, max = 66.40, var = 1.53 | |
| 1.41.636.830 I 02217: f32: norm-54 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.35, avg = -0.01, max = 12.80, var = 1.00 | |
| 1.41.651.826 I 02218: f32: Kcur_normed-54 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = -0.00, max = 1.59, var = 0.02 | |
| 1.41.667.052 I 02219: f32: Kcur_pos-54 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = -0.00, max = 1.59, var = 0.02 | |
| 1.41.701.234 I 02220: f32: Vcur-54 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -384.21, avg = 0.01, max = 388.00, var = 2.31 | |
| 1.41.716.349 I 02221: f32: Vcur-54 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -384.21, avg = 0.01, max = 388.00, var = 2.31 | |
| 1.41.731.653 I 02222: f32: Vcur_normed-54 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.41.746.344 I 02223: f32: Kcur_pos-54 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = -0.00, max = 1.59, var = 0.02 | |
| 1.41.790.434 I 02224: f16: cache_k_l54 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = -0.00, max = 1.59, var = 0.01 | |
| 1.41.805.082 I 02225: f32: Vcur_normed-54 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.41.849.308 I 02226: f16: cache_v_l54 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 0.33 | |
| 1.41.878.573 I 02227: f32: Qcur_pos-54 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = 0.00, max = 13.69, var = 1.02 | |
| 1.41.907.832 I 02228: f32: Qcur_pos-54 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = 0.00, max = 13.69, var = 1.02 | |
| 1.41.922.625 I 02229: f16: cache_k_l54 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = -0.00, max = 1.59, var = 0.02 | |
| 1.41.938.110 I 02230: f16: cache_k_l54 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.53, avg = -0.00, max = 1.59, var = 0.02 | |
| 1.41.952.666 I 02231: f16: cache_v_l54 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.41.967.214 I 02232: f16: cache_v_l54 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.42.011.338 I 02233: f32: __fattn__-54 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.99, var = 0.21 | |
| 1.42.041.607 I 02234: f32: kqv_out-54 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.99, var = 0.21 | |
| 1.42.098.759 I 02235: f32: node_2234 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.18, avg = 0.01, max = 15.90, var = 0.35 | |
| 1.42.118.746 I 02236: f32: norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -24.53, avg = 0.02, max = 28.61, var = 1.00 | |
| 1.42.138.364 I 02237: f32: attn_post_norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -2.13, avg = 0.00, max = 6.53, var = 0.04 | |
| 1.42.158.078 I 02238: f32: attn_out-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.46, avg = 0.01, max = 84.96, var = 1.23 | |
| 1.42.177.988 I 02239: f32: norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.57, avg = 0.01, max = 68.77, var = 1.00 | |
| 1.42.197.539 I 02240: f32: ffn_norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.81, avg = -0.00, max = 36.91, var = 0.45 | |
| 1.42.373.036 I 02241: f32: ffn_gate-54 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.71, avg = -1.55, max = 41.34, var = 0.95 | |
| 1.42.549.191 I 02242: f32: ffn_up-54 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.00, avg = 0.00, max = 43.30, var = 1.04 | |
| 1.42.630.183 I 02243: f32: ffn_geglu-54 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -420.52, avg = 0.00, max = 77.23, var = 0.11 | |
| 1.42.746.871 I 02244: f32: ffn_out-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.78, avg = -0.01, max = 56.56, var = 1.11 | |
| 1.42.766.930 I 02245: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.96, avg = -0.01, max = 22.90, var = 1.00 | |
| 1.42.786.742 I 02246: f32: ffn_post_norm-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -4.31, avg = 0.00, max = 18.70, var = 0.13 | |
| 1.42.806.435 I 02247: f32: node_2246 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.91, avg = 0.01, max = 94.05, var = 1.51 | |
| 1.42.826.426 I 02248: f32: l_out-54 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.87, avg = 0.01, max = 84.86, var = 1.23 | |
| 1.42.846.343 I 02249: f32: norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.18, avg = 0.01, max = 69.53, var = 1.00 | |
| 1.42.865.891 I 02250: f32: attn_norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -25.31, avg = -0.01, max = 123.70, var = 0.81 | |
| 1.42.933.071 I 02251: f32: Qcur-55 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.54, avg = 0.02, max = 34.02, var = 2.08 | |
| 1.42.962.423 I 02252: f32: Qcur-55 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -33.54, avg = 0.02, max = 34.02, var = 2.08 | |
| 1.42.992.841 I 02253: f32: norm-55 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.23, avg = 0.02, max = 13.87, var = 1.00 | |
| 1.43.022.510 I 02254: f32: Qcur_normed-55 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.57, avg = 0.02, max = 14.19, var = 1.05 | |
| 1.43.052.623 I 02255: f32: Qcur_pos-55 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.92, avg = 0.00, max = 14.00, var = 1.05 | |
| 1.43.086.794 I 02256: f32: Kcur-55 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.79, avg = -0.00, max = 82.58, var = 2.18 | |
| 1.43.101.902 I 02257: f32: Kcur-55 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -96.79, avg = -0.00, max = 82.58, var = 2.18 | |
| 1.43.117.203 I 02258: f32: norm-55 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.85, avg = -0.00, max = 13.01, var = 1.00 | |
| 1.43.132.156 I 02259: f32: Kcur_normed-55 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.59, var = 0.01 | |
| 1.43.147.307 I 02260: f32: Kcur_pos-55 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.59, var = 0.01 | |
| 1.43.181.253 I 02261: f32: Vcur-55 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -424.21, avg = 0.01, max = 519.06, var = 3.18 | |
| 1.43.196.369 I 02262: f32: Vcur-55 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -424.21, avg = 0.01, max = 519.06, var = 3.18 | |
| 1.43.211.684 I 02263: f32: Vcur_normed-55 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.43.226.375 I 02264: f32: Kcur_pos-55 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.59, var = 0.01 | |
| 1.43.270.443 I 02265: f16: cache_k_l55 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.59, var = 0.00 | |
| 1.43.285.100 I 02266: f32: Vcur_normed-55 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.43.329.053 I 02267: f16: cache_v_l55 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 0.33 | |
| 1.43.358.324 I 02268: f32: Qcur_pos-55 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.92, avg = 0.00, max = 14.00, var = 1.05 | |
| 1.43.387.606 I 02269: f32: Qcur_pos-55 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.92, avg = 0.00, max = 14.00, var = 1.05 | |
| 1.43.402.215 I 02270: f16: cache_k_l55 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.59, var = 0.01 | |
| 1.43.416.827 I 02271: f16: cache_k_l55 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.59, var = 0.01 | |
| 1.43.431.376 I 02272: f16: cache_v_l55 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.43.445.907 I 02273: f16: cache_v_l55 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.43.485.728 I 02274: f32: __fattn__-55 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.02, max = 15.93, var = 0.20 | |
| 1.43.515.144 I 02275: f32: kqv_out-55 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.02, max = 15.93, var = 0.20 | |
| 1.43.572.420 I 02276: f32: node_2275 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.32, avg = 0.01, max = 13.16, var = 0.48 | |
| 1.43.592.294 I 02277: f32: norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.18, avg = 0.02, max = 13.40, var = 1.00 | |
| 1.43.611.872 I 02278: f32: attn_post_norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -2.34, avg = 0.00, max = 7.43, var = 0.06 | |
| 1.43.631.605 I 02279: f32: attn_out-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.65, avg = 0.01, max = 91.61, var = 1.30 | |
| 1.43.651.682 I 02280: f32: norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.51, avg = 0.01, max = 69.66, var = 1.00 | |
| 1.43.671.336 I 02281: f32: ffn_norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.54, avg = -0.00, max = 42.37, var = 0.36 | |
| 1.43.846.616 I 02282: f32: ffn_gate-55 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.15, avg = -1.14, max = 19.00, var = 0.85 | |
| 1.44.022.294 I 02283: f32: ffn_up-55 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -52.75, avg = 0.00, max = 26.02, var = 0.62 | |
| 1.44.103.467 I 02284: f32: ffn_geglu-55 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -458.26, avg = -0.00, max = 155.43, var = 0.10 | |
| 1.44.220.319 I 02285: f32: ffn_out-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.29, avg = 0.01, max = 50.95, var = 0.92 | |
| 1.44.240.335 I 02286: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.59, avg = 0.01, max = 17.11, var = 1.00 | |
| 1.44.259.941 I 02287: f32: ffn_post_norm-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.07, avg = 0.00, max = 15.38, var = 0.17 | |
| 1.44.279.702 I 02288: f32: node_2287 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.91, avg = 0.01, max = 104.81, var = 1.60 | |
| 1.44.299.660 I 02289: f32: l_out-55 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.81, avg = 0.01, max = 92.53, var = 1.25 | |
| 1.44.319.562 I 02290: f32: norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.74, avg = 0.01, max = 69.38, var = 1.00 | |
| 1.44.338.915 I 02291: f32: attn_norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.83, avg = -0.01, max = 82.34, var = 0.53 | |
| 1.44.406.529 I 02292: f32: Qcur-56 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.10, avg = 0.02, max = 27.31, var = 1.44 | |
| 1.44.436.736 I 02293: f32: Qcur-56 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -41.10, avg = 0.02, max = 27.31, var = 1.44 | |
| 1.44.467.074 I 02294: f32: norm-56 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.62, avg = 0.02, max = 13.97, var = 1.00 | |
| 1.44.496.755 I 02295: f32: Qcur_normed-56 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.83, avg = 0.02, max = 14.19, var = 1.03 | |
| 1.44.526.725 I 02296: f32: Qcur_pos-56 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.86, avg = 0.02, max = 14.41, var = 1.03 | |
| 1.44.560.977 I 02297: f32: Kcur-56 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -73.66, avg = 0.01, max = 78.36, var = 1.63 | |
| 1.44.576.089 I 02298: f32: Kcur-56 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -73.66, avg = 0.01, max = 78.36, var = 1.63 | |
| 1.44.591.221 I 02299: f32: norm-56 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.30, avg = 0.01, max = 14.21, var = 1.00 | |
| 1.44.606.214 I 02300: f32: Kcur_normed-56 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.39, avg = 0.00, max = 1.75, var = 0.02 | |
| 1.44.621.181 I 02301: f32: Kcur_pos-56 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.62, avg = 0.00, max = 1.74, var = 0.02 | |
| 1.44.655.172 I 02302: f32: Vcur-56 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -506.42, avg = 0.00, max = 467.85, var = 2.61 | |
| 1.44.670.287 I 02303: f32: Vcur-56 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -506.42, avg = 0.00, max = 467.85, var = 2.61 | |
| 1.44.685.567 I 02304: f32: Vcur_normed-56 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.44.700.255 I 02305: f32: Kcur_pos-56 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.62, avg = 0.00, max = 1.74, var = 0.02 | |
| 1.44.744.363 I 02306: f16: cache_k_l56 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.62, avg = 0.00, max = 1.74, var = 0.01 | |
| 1.44.759.018 I 02307: f32: Vcur_normed-56 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.00, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.44.802.973 I 02308: f16: cache_v_l56 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.00, max = 15.99, var = 0.33 | |
| 1.44.832.291 I 02309: f32: Qcur_pos-56 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.86, avg = 0.02, max = 14.41, var = 1.03 | |
| 1.44.861.565 I 02310: f32: Qcur_pos-56 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.86, avg = 0.02, max = 14.41, var = 1.03 | |
| 1.44.876.208 I 02311: f16: cache_k_l56 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.62, avg = 0.00, max = 1.74, var = 0.02 | |
| 1.44.890.830 I 02312: f16: cache_k_l56 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.62, avg = 0.00, max = 1.74, var = 0.02 | |
| 1.44.905.413 I 02313: f16: cache_v_l56 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.44.919.963 I 02314: f16: cache_v_l56 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = 0.01, max = 15.99, var = 1.00 | |
| 1.44.960.114 I 02315: f32: __fattn__-56 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 0.19 | |
| 1.44.989.561 I 02316: f32: kqv_out-56 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.99, avg = -0.01, max = 15.99, var = 0.19 | |
| 1.45.047.046 I 02317: f32: node_2316 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.75, avg = -0.01, max = 11.70, var = 0.49 | |
| 1.45.066.937 I 02318: f32: norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.66, avg = -0.01, max = 22.23, var = 1.00 | |
| 1.45.086.504 I 02319: f32: attn_post_norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -3.04, avg = 0.00, max = 3.93, var = 0.06 | |
| 1.45.106.350 I 02320: f32: attn_out-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.02, avg = 0.01, max = 94.71, var = 1.30 | |
| 1.45.126.278 I 02321: f32: norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.28, avg = 0.01, max = 68.61, var = 1.00 | |
| 1.45.145.812 I 02322: f32: ffn_norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.83, avg = -0.00, max = 63.98, var = 0.43 | |
| 1.45.321.577 I 02323: f32: ffn_gate-56 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.34, avg = -1.29, max = 10.30, var = 0.92 | |
| 1.45.497.161 I 02324: f32: ffn_up-56 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -71.42, avg = -0.01, max = 36.53, var = 0.79 | |
| 1.45.578.074 I 02325: f32: ffn_geglu-56 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -730.78, avg = -0.00, max = 51.61, var = 0.11 | |
| 1.45.694.866 I 02326: f32: ffn_out-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.30, avg = -0.01, max = 73.54, var = 0.88 | |
| 1.45.714.769 I 02327: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.07, avg = -0.01, max = 15.86, var = 1.00 | |
| 1.45.734.458 I 02328: f32: ffn_post_norm-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.07, avg = -0.00, max = 13.81, var = 0.21 | |
| 1.45.754.149 I 02329: f32: node_2328 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.75, avg = 0.01, max = 82.84, var = 1.32 | |
| 1.45.774.140 I 02330: f32: l_out-56 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.05, avg = 0.01, max = 71.84, var = 0.99 | |
| 1.45.794.057 I 02331: f32: norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.90, avg = 0.01, max = 66.11, var = 1.00 | |
| 1.45.813.589 I 02332: f32: attn_norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.83, avg = -0.00, max = 132.36, var = 0.55 | |
| 1.45.881.037 I 02333: f32: Qcur-57 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.93, avg = -0.03, max = 24.90, var = 1.86 | |
| 1.45.911.241 I 02334: f32: Qcur-57 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -29.93, avg = -0.03, max = 24.90, var = 1.86 | |
| 1.45.942.218 I 02335: f32: norm-57 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.68, avg = -0.02, max = 12.08, var = 1.00 | |
| 1.45.973.084 I 02336: f32: Qcur_normed-57 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.28, avg = -0.02, max = 12.64, var = 1.10 | |
| 1.46.004.198 I 02337: f32: Qcur_pos-57 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.39, avg = 0.00, max = 12.64, var = 1.10 | |
| 1.46.047.758 I 02338: f32: Kcur-57 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -64.64, avg = -0.01, max = 49.40, var = 2.01 | |
| 1.46.062.868 I 02339: f32: Kcur-57 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -64.64, avg = -0.01, max = 49.40, var = 2.01 | |
| 1.46.078.425 I 02340: f32: norm-57 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.02, avg = -0.01, max = 9.98, var = 1.00 | |
| 1.46.094.098 I 02341: f32: Kcur_normed-57 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.44, avg = -0.00, max = 1.19, var = 0.01 | |
| 1.46.109.313 I 02342: f32: Kcur_pos-57 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.23, var = 0.01 | |
| 1.46.143.491 I 02343: f32: Vcur-57 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -347.76, avg = -0.00, max = 368.15, var = 1.64 | |
| 1.46.158.724 I 02344: f32: Vcur-57 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -347.76, avg = -0.00, max = 368.15, var = 1.64 | |
| 1.46.174.365 I 02345: f32: Vcur_normed-57 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.46.189.504 I 02346: f32: Kcur_pos-57 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.23, var = 0.01 | |
| 1.46.233.458 I 02347: f16: cache_k_l57 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.23, var = 0.00 | |
| 1.46.248.408 I 02348: f32: Vcur_normed-57 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.46.292.189 I 02349: f16: cache_v_l57 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 0.33 | |
| 1.46.322.095 I 02350: f32: Qcur_pos-57 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.39, avg = 0.00, max = 12.64, var = 1.10 | |
| 1.46.352.293 I 02351: f32: Qcur_pos-57 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.39, avg = 0.00, max = 12.64, var = 1.10 | |
| 1.46.366.958 I 02352: f16: cache_k_l57 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.23, var = 0.01 | |
| 1.46.381.600 I 02353: f16: cache_k_l57 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.45, avg = -0.00, max = 1.23, var = 0.01 | |
| 1.46.396.145 I 02354: f16: cache_v_l57 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.46.410.693 I 02355: f16: cache_v_l57 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.46.451.368 I 02356: f32: __fattn__-57 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 0.21 | |
| 1.46.481.629 I 02357: f32: kqv_out-57 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = 0.00, max = 15.98, var = 0.21 | |
| 1.46.538.552 I 02358: f32: node_2357 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.89, avg = 0.01, max = 9.46, var = 0.57 | |
| 1.46.558.916 I 02359: f32: norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -27.64, avg = 0.01, max = 10.05, var = 1.00 | |
| 1.46.579.018 I 02360: f32: attn_post_norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.40, avg = 0.00, max = 5.17, var = 0.16 | |
| 1.46.599.276 I 02361: f32: attn_out-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.34, avg = 0.01, max = 66.29, var = 1.02 | |
| 1.46.619.679 I 02362: f32: norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.24, avg = 0.01, max = 60.23, var = 1.00 | |
| 1.46.639.745 I 02363: f32: ffn_norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.18, avg = -0.00, max = 62.24, var = 0.44 | |
| 1.46.815.309 I 02364: f32: ffn_gate-57 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.75, avg = -1.46, max = 14.06, var = 1.01 | |
| 1.46.990.928 I 02365: f32: ffn_up-57 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -26.46, avg = 0.00, max = 37.54, var = 0.87 | |
| 1.47.071.854 I 02366: f32: ffn_geglu-57 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -231.96, avg = -0.00, max = 83.11, var = 0.14 | |
| 1.47.188.631 I 02367: f32: ffn_out-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.21, avg = -0.03, max = 21.64, var = 1.54 | |
| 1.47.209.082 I 02368: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -40.59, avg = -0.03, max = 12.10, var = 1.00 | |
| 1.47.229.183 I 02369: f32: ffn_post_norm-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -49.79, avg = -0.00, max = 8.50, var = 0.48 | |
| 1.47.249.432 I 02370: f32: node_2369 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.16, avg = 0.01, max = 56.41, var = 0.93 | |
| 1.47.270.015 I 02371: f32: l_out-57 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.07, avg = 0.01, max = 45.61, var = 0.61 | |
| 1.47.290.445 I 02372: f32: norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.90, avg = 0.01, max = 29.81, var = 1.00 | |
| 1.47.310.528 I 02373: f32: attn_norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -32.24, avg = -0.01, max = 161.16, var = 0.61 | |
| 1.47.377.675 I 02374: f32: Qcur-58 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.34, avg = 0.01, max = 28.15, var = 2.24 | |
| 1.47.407.878 I 02375: f32: Qcur-58 (reshaped) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -31.34, avg = 0.01, max = 28.15, var = 2.24 | |
| 1.47.439.030 I 02376: f32: norm-58 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.54, avg = 0.01, max = 13.75, var = 1.00 | |
| 1.47.469.688 I 02377: f32: Qcur_normed-58 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.86, avg = 0.01, max = 14.07, var = 1.05 | |
| 1.47.500.425 I 02378: f32: Qcur_pos-58 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.53, avg = -0.01, max = 13.43, var = 1.05 | |
| 1.47.534.269 I 02379: f32: Kcur-58 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.87, avg = 0.01, max = 58.05, var = 1.49 | |
| 1.47.549.376 I 02380: f32: Kcur-58 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -58.87, avg = 0.01, max = 58.05, var = 1.49 | |
| 1.47.565.183 I 02381: f32: norm-58 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.68, avg = 0.01, max = 11.60, var = 1.00 | |
| 1.47.580.218 I 02382: f32: Kcur_normed-58 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = 0.00, max = 1.42, var = 0.01 | |
| 1.47.596.242 I 02383: f32: Kcur_pos-58 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = 0.00, max = 1.42, var = 0.01 | |
| 1.47.632.139 I 02384: f32: Vcur-58 [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -350.87, avg = -0.01, max = 268.81, var = 1.42 | |
| 1.47.647.580 I 02385: f32: Vcur-58 (reshaped) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -350.87, avg = -0.01, max = 268.81, var = 1.42 | |
| 1.47.663.223 I 02386: f32: Vcur_normed-58 [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.47.678.362 I 02387: f32: Kcur_pos-58 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = 0.00, max = 1.42, var = 0.01 | |
| 1.47.722.403 I 02388: f16: cache_k_l58 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = 0.00, max = 1.42, var = 0.00 | |
| 1.47.737.193 I 02389: f32: Vcur_normed-58 (view) [ 4096, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.47.780.991 I 02390: f16: cache_v_l58 (view) [ 4096, 1536, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.00, max = 15.98, var = 0.33 | |
| 1.47.811.083 I 02391: f32: Qcur_pos-58 (view) [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.53, avg = -0.01, max = 13.43, var = 1.05 | |
| 1.47.841.285 I 02392: f32: Qcur_pos-58 (view) (permuted) [ 256, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.53, avg = -0.01, max = 13.43, var = 1.05 | |
| 1.47.855.965 I 02393: f16: cache_k_l58 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = 0.00, max = 1.42, var = 0.01 | |
| 1.47.870.580 I 02394: f16: cache_k_l58 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.55, avg = 0.00, max = 1.42, var = 0.01 | |
| 1.47.885.127 I 02395: f16: cache_v_l58 (view) [ 256, 16, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.47.899.682 I 02396: f16: cache_v_l58 (view) (permuted) [ 256, 512, 16, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.98, avg = -0.01, max = 15.98, var = 1.00 | |
| 1.47.940.958 I 02397: f32: __fattn__-58 [ 256, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.00, max = 15.98, var = 0.28 | |
| 1.47.971.207 I 02398: f32: kqv_out-58 [ 8192, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.95, avg = 0.00, max = 15.98, var = 0.28 | |
| 1.48.028.192 I 02399: f32: node_2398 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.31, avg = -0.00, max = 15.18, var = 0.50 | |
| 1.48.048.687 I 02400: f32: norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -22.70, avg = -0.00, max = 21.43, var = 1.00 | |
| 1.48.069.637 I 02401: f32: attn_post_norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.59, avg = 0.00, max = 22.27, var = 0.48 | |
| 1.48.090.692 I 02402: f32: attn_out-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.69, avg = 0.01, max = 33.00, var = 1.05 | |
| 1.48.110.987 I 02403: f32: norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.22, avg = 0.01, max = 22.21, var = 1.00 | |
| 1.48.131.107 I 02404: f32: ffn_norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.68, avg = 0.00, max = 19.03, var = 0.77 | |
| 1.48.310.284 I 02405: f32: ffn_gate-58 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -13.47, avg = -1.17, max = 26.00, var = 1.39 | |
| 1.48.497.662 I 02406: f32: ffn_up-58 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.98, avg = 0.01, max = 21.93, var = 1.49 | |
| 1.48.580.287 I 02407: f32: ffn_geglu-58 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -283.79, avg = -0.01, max = 77.61, var = 0.71 | |
| 1.48.698.285 I 02408: f32: ffn_out-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -38.87, avg = 0.01, max = 38.96, var = 5.77 | |
| 1.48.718.551 I 02409: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -10.05, avg = 0.01, max = 10.07, var = 1.00 | |
| 1.48.739.297 I 02410: f32: ffn_post_norm-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -9.90, avg = -0.01, max = 7.13, var = 0.67 | |
| 1.48.759.731 I 02411: f32: node_2410 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.64, avg = 0.01, max = 30.44, var = 1.50 | |
| 1.48.779.761 I 02412: f32: l_out-58 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -11.04, avg = 0.00, max = 22.95, var = 0.85 | |
| 1.48.801.467 I 02413: f32: norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.08, avg = 0.00, max = 20.06, var = 1.00 | |
| 1.48.821.746 I 02414: f32: attn_norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -37.41, avg = 0.00, max = 34.17, var = 2.29 | |
| 1.48.957.296 I 02415: f32: Qcur-59 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -87.39, avg = -0.00, max = 81.66, var = 2.35 | |
| 1.49.017.695 I 02416: f32: Qcur-59 (reshaped) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -87.39, avg = -0.00, max = 81.66, var = 2.35 | |
| 1.49.080.092 I 02417: f32: norm-59 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.07, avg = -0.00, max = 15.27, var = 1.00 | |
| 1.49.141.192 I 02418: f32: Qcur_normed-59 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.66, avg = -0.00, max = 15.87, var = 1.08 | |
| 1.49.202.624 I 02419: f32: Qcur_pos-59 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.66, avg = -0.00, max = 15.87, var = 1.08 | |
| 1.49.219.945 I 02420: f32: Vcur-59 [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -109.14, avg = 0.02, max = 160.31, var = 2.19 | |
| 1.49.227.497 I 02421: f32: Vcur-59 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -109.14, avg = 0.02, max = 160.31, var = 2.19 | |
| 1.49.235.608 I 02422: f32: norm-59 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 1.00 | |
| 1.49.243.179 I 02423: f32: Kcur_normed-59 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = 0.00, max = 1.33, var = 0.00 | |
| 1.49.250.805 I 02424: f32: Kcur_pos-59 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = 0.00, max = 1.33, var = 0.00 | |
| 1.49.258.179 I 02425: f32: Vcur-59 (reshaped) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -109.14, avg = 0.02, max = 160.31, var = 2.19 | |
| 1.49.265.940 I 02426: f32: Vcur_normed-59 [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 1.00 | |
| 1.49.273.705 I 02427: f32: Kcur_pos-59 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = 0.00, max = 1.33, var = 0.00 | |
| 1.49.332.011 I 02428: f16: cache_k_l59 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = 0.00, max = 1.33, var = 0.00 | |
| 1.49.339.342 I 02429: f32: Vcur_normed-59 (view) [ 2048, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 1.00 | |
| 1.49.397.709 I 02430: f16: cache_v_l59 (view) [ 2048, 4096, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.00, max = 22.11, var = 0.12 | |
| 1.49.456.308 I 02431: f32: Qcur_pos-59 (view) [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.66, avg = -0.00, max = 15.87, var = 1.08 | |
| 1.49.514.831 I 02432: f32: Qcur_pos-59 (view) (permuted) [ 512, 512, 32, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.66, avg = -0.00, max = 15.87, var = 1.08 | |
| 1.49.522.155 I 02433: f16: cache_k_l59 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = 0.00, max = 1.33, var = 0.00 | |
| 1.49.529.498 I 02434: f16: cache_k_l59 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -1.31, avg = 0.00, max = 1.33, var = 0.00 | |
| 1.49.536.774 I 02435: f16: cache_v_l59 (view) [ 512, 4, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 1.00 | |
| 1.49.544.049 I 02436: f16: cache_v_l59 (view) (permuted) [ 512, 512, 4, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 1.00 | |
| 1.49.630.916 I 02437: f32: __fattn__-59 [ 512, 32, 512, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 0.54 | |
| 1.49.691.339 I 02438: f32: kqv_out-59 [ 16384, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -21.84, avg = 0.01, max = 22.11, var = 0.54 | |
| 1.49.786.555 I 02439: f32: node_2438 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -23.98, avg = -0.02, max = 15.95, var = 1.07 | |
| 1.49.807.186 I 02440: f32: norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -16.24, avg = -0.02, max = 17.37, var = 1.00 | |
| 1.49.827.523 I 02441: f32: attn_post_norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.67, avg = -0.03, max = 26.60, var = 1.52 | |
| 1.49.847.677 I 02442: f32: attn_out-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -20.95, avg = -0.02, max = 25.79, var = 1.94 | |
| 1.49.868.075 I 02443: f32: norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -14.83, avg = -0.02, max = 16.97, var = 1.00 | |
| 1.49.888.408 I 02444: f32: ffn_norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -19.98, avg = -0.00, max = 13.02, var = 1.09 | |
| 1.50.065.179 I 02445: f32: ffn_gate-59 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -15.50, avg = -0.91, max = 14.45, var = 1.98 | |
| 1.50.249.571 I 02446: f32: ffn_up-59 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -18.81, avg = 0.00, max = 18.77, var = 1.52 | |
| 1.50.333.233 I 02447: f32: ffn_geglu-59 [ 21504, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -271.78, avg = -0.00, max = 42.78, var = 0.84 | |
| 1.50.456.226 I 02448: f32: ffn_out-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -46.15, avg = 0.01, max = 27.78, var = 3.21 | |
| 1.50.476.519 I 02449: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -17.48, avg = 0.00, max = 11.95, var = 1.00 | |
| 1.50.497.238 I 02450: f32: ffn_post_norm-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -346.98, avg = -0.00, max = 415.26, var = 16.02 | |
| 1.50.517.785 I 02451: f32: node_2450 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -345.62, avg = -0.03, max = 414.73, var = 16.06 | |
| 1.50.537.908 I 02452: f32: l_out-59 [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -12.57, avg = -0.00, max = 15.09, var = 0.02 | |
| 1.50.559.515 I 02453: f32: norm [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -64.90, avg = -0.01, max = 64.03, var = 1.00 | |
| 1.50.580.355 I 02454: f32: h_nextn [ 5376, 512, 1, 1 ] all zero = 0, infs = 0, nans = 0, min = -258.41, avg = -0.03, max = 147.91, var = 16.65 | |
| 1.50.580.377 I run_debug: done. | |
| 1.50.580.382 I print_report: n_zero_tensors: 0 | |
| 1.50.580.383 I print_report: n_inf_tensors: 0 | |
| 1.50.580.383 I print_report: n_nan_tensors: 0 | |
| 1.50.580.383 I print_report: n_tensors: 2454 | |
| 1.50.580.384 I print_report: total tensor data captured: 32632.00 MiB |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment