dpo.yml

# 主要改进
# - 换成 Full finetuning
# Model define
base_model: /cephfs/panwenbo/work/models/Faro-34B
model_type: LlamaForCausalLM
tokenizer_type: LlamaTokenizer

# is_qwen_derived_model: true
trust_remote_code: true
chat_template: chatml

# Data
rl: dpo
datasets:
  - path: argilla/ultrafeedback-binarized-preferences-cleaned
    split: train
    type: chatml.ultra
  - path: Intel/orca_dpo_pairs
    split: train
    type: chatml.intel
  - path: jondurbin/truthy-dpo-v0.1
    split: train
    type: chatml.prompt_pairs
  - path: wenbopan/Chinese-dpo-pairs
    split: train
    type: chatml.prompt_pairs
  - path: wenbopan/cmmlu_dpo_pairs
    split: train
    type: chatml.prompt_pairs
dataset_prepared_path: data/prepared
val_set_size: 0.05
output_dir: ./outputs/fi-dpo-34/

sequence_len: 4096
sample_packing: false
pad_to_sequence_len: true
eval_sample_packing: false
group_by_length: true
s2_attention: 

# Training config
load_in_8bit: false
load_in_4bit: false
strict: false

# Lora
adapter: lora
lora_model_dir: 
lora_r: 64
lora_alpha: 64
lora_dropout: 0.05
lora_target_linear: true
lora_fan_in_fan_out:
# lora_modules_to_save:
#   - embed_tokens
#   - lm_head
#   - norm

# Wandb Log
wandb_project: Faro-DPO
wandb_entity: bmpixel
wandb_watch:
wandb_name: fi-34
wandb_log_model:

# Training Optimization
gradient_accumulation_steps: 4
micro_batch_size: 3
num_epochs: 2
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 0.0001
max_grad_norm: 1.0

train_on_inputs: false
bf16: true
fp16: 
tf32: false

flash_optimum: 
gradient_checkpointing: true
early_stopping_patience:
auto_resume_from_checkpoints: false
local_rank:
logging_steps: 1
xformers_attention: false
flash_attention: true

warmup_steps: 50
evals_per_epoch: 2
eval_table_size:
eval_max_new_tokens: 512
saves_per_epoch: 2
# save_steps: 2
weight_decay: 0.0