4B SFT long context

config.yaml

# Model arguments
model_name_or_path: Qwen/Qwen3-4B-Instruct-2507
model_revision: main
attn_implementation: kernels-community/vllm-flash-attn3

# Data training arguments
dataset_mixture:
  datasets:
    - id: hf-carbon/seqqa-sft-v1_gemma-4-31B-it
      config: graded_correct
      split: train            
      columns:
        - messages
      weight: 1.0
  seed: 0
dataset_num_proc: 12

# SFT trainer config
assistant_only_loss: false
bf16: true
eval_strategy: 'no'
gradient_accumulation_steps: 2
gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false
learning_rate: 1.0e-06
log_level: info
logging_steps: 1
logging_strategy: steps
lr_scheduler_type: cosine_with_min_lr
lr_scheduler_kwargs:
  min_lr_rate: 0.1
max_grad_norm: 0.2
max_length: 36864
num_train_epochs: 5
output_dir: data/Qwen3-4B-Instruct-2507-SFT
packing: true
per_device_train_batch_size: 2
per_device_eval_batch_size: 2
push_to_hub: true
report_to:
- wandb
save_strategy: epoch
save_total_limit: 1
seed: 42
use_liger_kernel: true
warmup_ratio: 0.03
wandb_entity: huggingface

DeepSpeed config

compute_environment: LOCAL_MACHINE
debug: false
deepspeed_config:
  deepspeed_multinode_launcher: standard
  offload_optimizer_device: none
  offload_param_device: none
  zero3_init_flag: true
  zero3_save_16bit_model: true
  zero_stage: 3
distributed_type: DEEPSPEED
downcast_bf16: 'no'
machine_rank: 0
main_training_function: main
mixed_precision: bf16
num_machines: 1
num_processes: 8
rdzv_backend: static
same_network: true
tpu_env: []
tpu_use_cluster: false
tpu_use_sudo: false
use_cpu: false