Faro Yi 9B config

fi.yml

# 主要改进
# - 换成 Full finetuning
# Model define
base_model: /cephfs/panwenbo/work/models/Yi-9B-200K
model_type: LlamaForCausalLM
tokenizer_type: LlamaTokenizer

# is_qwen_derived_model: true
trust_remote_code: true
chat_template: chatml

# Data
datasets:
  - path: wenbopan/Fusang-v1
    name: base
    type: sharegpt
  - path: wenbopan/Fusang-v1
    name: long
    type: sharegpt
  - path: wenbopan/OpenOrca-zh-20k
    name: zh
    type: alpaca_w_system.load_open_orca
  - path: wenbopan/OpenOrca-zh-20k
    name: en
    type: alpaca_w_system.load_open_orca
dataset_prepared_path: data/prepared
val_set_size: 0.05
output_dir: ./outputs/fi-chatml/

sequence_len: 24576
sample_packing: true
pad_to_sequence_len: true
eval_sample_packing: true
group_by_length: true
s2_attention: 

# Training config
load_in_8bit: false
load_in_4bit: false
strict: false

# Lora
adapter: lora
lora_model_dir: 
lora_r: 32
lora_alpha: 64
lora_dropout: 0.05
lora_target_linear: true
lora_fan_in_fan_out:
# lora_modules_to_save:
#   - embed_tokens
#   - lm_head
#   - norm

# Wandb Log
wandb_project: mixtral
wandb_entity: bmpixel
wandb_watch:
wandb_name: fi-chatml
wandb_log_model:

# Training Optimization
gradient_accumulation_steps: 2
micro_batch_size: 2
num_epochs: 2
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 0.0001
max_grad_norm: 1.0

train_on_inputs: false
bf16: true
fp16: 
tf32: false

flash_optimum: 
gradient_checkpointing: true
early_stopping_patience:
auto_resume_from_checkpoints: false
local_rank:
logging_steps: 1
xformers_attention: false
flash_attention: true

warmup_steps: 50
evals_per_epoch: 4
eval_table_size:
eval_max_new_tokens: 512
saves_per_epoch: 4
# save_steps: 2
weight_decay: 0.0