PyTorch autograd and linear regression examples

gistfile1.txt

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ba091f7e",
   "metadata": {},
   "source": [
    "# CS 231n example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "71e4f036",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8e2a3e31",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.)\n",
      "tensor(-4.)\n"
     ]
    }
   ],
   "source": [
    "x=torch.tensor(-2.,requires_grad=True)\n",
    "y=torch.tensor(5.,requires_grad=True)\n",
    "z=torch.tensor(-4.,requires_grad=True)\n",
    "\n",
    "q=x+y\n",
    "\n",
    "f=q*z\n",
    "\n",
    "f.backward()\n",
    "\n",
    "print(z.grad)\n",
    "print(y.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3aa2c526",
   "metadata": {},
   "source": [
    "## Another example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6e94a955",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.3932)\n",
      "tensor(-0.1966)\n"
     ]
    }
   ],
   "source": [
    "w0=torch.tensor(2.,requires_grad=True)\n",
    "x0=torch.tensor(-1.,requires_grad=True)\n",
    "w1=torch.tensor(-3.,requires_grad=True)\n",
    "x1=torch.tensor(-2.,requires_grad=True)\n",
    "w2=torch.tensor(-3.)\n",
    "\n",
    "f=1/(1+torch.exp(-(w0*x0+w1*x1+w2)))\n",
    "\n",
    "f.backward()\n",
    "\n",
    "print(x0.grad)\n",
    "print(w0.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "541302f9",
   "metadata": {},
   "source": [
    "# Linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "676cd4c4",
   "metadata": {},
   "source": [
    "## create data and targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "89d1ad6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.optim as optim\n",
    "\n",
    "def linear_model(x, W, b):\n",
    "    return torch.matmul(x, W) + b\n",
    "\n",
    "Wt =torch.tensor([[1.,2.,3.],[4.,5.,6.]])\n",
    "bt = torch.tensor([1.,2.,3.])\n",
    "data = torch.randn(10000,2)\n",
    "targets = torch.matmul(data,Wt)+bt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3ad97979",
   "metadata": {},
   "source": [
    "## without optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b1c944ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.0000, 2.0000, 3.0000],\n",
      "        [4.0000, 5.0000, 6.0000]], requires_grad=True) tensor([1.0000, 2.0000, 3.0000], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "W = torch.randn(2, 3, requires_grad=True) # For earlier pytorch version, torch.tensor does not have requires_grad\n",
    "                                          # and need to use torch.autograd.Variable instead\n",
    "b = torch.randn(3, requires_grad=True)\n",
    "\n",
    "learning_rate=0.001\n",
    "for sample, target in zip(data, targets):\n",
    "    # clear out the gradients of Variables \n",
    "    # (i.e. W, b)\n",
    "    output = linear_model(sample, W, b)\n",
    "    loss = torch.sum((output - target) ** 2)\n",
    "    loss.backward()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        W -= learning_rate * W.grad.data\n",
    "        b -= learning_rate * b.grad.data\n",
    "\n",
    "#         W.grad.data.zero_()\n",
    "#         b.grad.data.zero_()\n",
    "        W.grad=None\n",
    "        b.grad=None\n",
    "\n",
    "\n",
    "print(W,b)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c227a4f2",
   "metadata": {},
   "source": [
    "## with optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "10c6a300",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.0000, 2.0000, 3.0000],\n",
      "        [4.0000, 5.0000, 6.0000]], requires_grad=True) tensor([1.0000, 2.0000, 3.0000], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "W = torch.randn(2, 3, requires_grad=True)\n",
    "b = torch.randn(3, requires_grad=True)\n",
    "\n",
    "optimizer = optim.SGD([W, b],lr=0.001)\n",
    "\n",
    "for sample, target in zip(data, targets):\n",
    "    # clear out the gradients of all Variables \n",
    "    # in this optimizer (i.e. W, b)\n",
    "    optimizer.zero_grad()\n",
    "    output = linear_model(sample, W, b)\n",
    "    loss = torch.sum((output - target) ** 2)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "print(W,b)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8b2b027",
   "metadata": {},
   "source": [
    "# For earlier PyTorch version, use torch.autograd.Variable instead"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f903f27",
   "metadata": {},
   "source": [
    "## Without optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2a8ad4d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.0000, 2.0000, 3.0000],\n",
      "        [4.0000, 5.0000, 6.0000]], requires_grad=True) tensor([1.0000, 2.0000, 3.0000], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "from torch.autograd import Variable # older version needs variable. For newer version, you may use Tensor directly\n",
    "\n",
    "W = Variable(torch.randn(2, 3), requires_grad=True)\n",
    "b = Variable(torch.randn(3), requires_grad=True)\n",
    "\n",
    "learning_rate=0.001\n",
    "for sample, target in zip(data, targets):\n",
    "    # clear out the gradients of Variables \n",
    "    # (i.e. W, b)\n",
    "    output = linear_model(sample, W, b)\n",
    "    loss = torch.sum((output - target) ** 2)\n",
    "    loss.backward()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        W -= learning_rate * W.grad.data\n",
    "        b -= learning_rate * b.grad.data\n",
    "\n",
    "#         W.grad.data.zero_()\n",
    "#         b.grad.data.zero_()\n",
    "        W.grad=None\n",
    "        b.grad=None\n",
    "\n",
    "\n",
    "print(W,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a836899d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.0000, 2.0000, 3.0000],\n",
      "        [4.0000, 5.0000, 6.0000]], requires_grad=True) tensor([1.0000, 2.0000, 3.0000], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "W = Variable(torch.randn(2, 3), requires_grad=True)\n",
    "b = Variable(torch.randn(3), requires_grad=True)\n",
    "\n",
    "optimizer = optim.SGD([W, b],lr=0.001)\n",
    "\n",
    "for sample, target in zip(data, targets):\n",
    "    # clear out the gradients of all Variables \n",
    "    # in this optimizer (i.e. W, b)\n",
    "    optimizer.zero_grad()\n",
    "    output = linear_model(sample, W, b)\n",
    "    loss = torch.sum((output - target) ** 2)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n",
    "print(W,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19134384",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a519bd0b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}