Minimal Merge

ultra-minimal-merge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12f39541-04a5-4e91-84c2-e798cdb43b53",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import PeftModel\n",
    "import torch\n",
    "\n",
    "import os\n",
    "\n",
    "def merge_lora(base_model_path, lora_path, output_dir):\n",
    "    device_arg = { 'device_map': 'auto' }\n",
    "    \n",
    "    base_model = AutoModelForCausalLM.from_pretrained(\n",
    "        base_model_path,\n",
    "        return_dict=True,\n",
    "        torch_dtype=torch.float16,\n",
    "        **device_arg\n",
    "    )\n",
    "\n",
    "    print(f\"Loading PEFT: {lora_path}\")\n",
    "    model = PeftModel.from_pretrained(base_model, lora_path, **device_arg)\n",
    "    print(f\"Running merge_and_unload\")\n",
    "    model = model.merge_and_unload()\n",
    "\n",
    "    tokenizer = AutoTokenizer.from_pretrained(base_model_path)\n",
    "\n",
    "    model.save_pretrained(output_dir)\n",
    "    tokenizer.save_pretrained(output_dir)\n",
    "    print(f\"Model saved to {output_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0388a4c4-71b6-4eed-ad11-1fa96b1be0a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "base = \"models/llama-2-13b\"\n",
    "lora = \"loras/llama-2-story-2\"\n",
    "out = \"merged\"\n",
    "merge_lora(base, lora, out)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}