zhezh · April 8, 2020 23:07
diff --git a/per-layer-lr.ipynb b/per-layer-lr.ipynb
 {
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "分层设置学习率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4.0\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建一个简单多层网络结构\n",
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        \"\"\"\n",
    "        In the constructor we instantiate two nn.Linear modules and assign them as\n",
    "        member variables.\n",
    "        \"\"\"\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, H)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        In the forward function we accept a Tensor of input data and we must return\n",
    "        a Tensor of output data. We can use Modules defined in the constructor as\n",
    "        well as arbitrary operators on Tensors.\n",
    "        \"\"\"\n",
    "        h_relu = F.relu(self.linear1(x))\n",
    "        y_pred = self.linear2(h_relu)\n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "# Construct our model by instantiating the class defined above\n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    "\n",
    "# Construct our loss function and an Optimizer. The call to model.parameters()\n",
    "# in the SGD constructor will contain the learnable parameters of the two\n",
    "# nn.Linear modules which are members of the model.\n",
    "criterion = torch.nn.MSELoss(size_average=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查看模型的参数名称"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "参数名: linear1.weight    , id: 140705409071936\n",
      "参数名: linear1.bias      , id: 140705409072008\n",
      "参数名: linear2.weight    , id: 140705409072296\n",
      "参数名: linear2.bias      , id: 140705409072656\n"
     ]
    }
   ],
   "source": [
    "for pname, p in model.named_parameters():\n",
    "    print('参数名: {: <18}, id: {}'.format(pname, id(p)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_parameters = model.parameters()\n",
    "\n",
    "lin1_parameters = []\n",
    "for pname, p in model.named_parameters():\n",
    "    if pname.find('linear1') >= 0:\n",
    "        lin1_parameters.append(p)\n",
    "\n",
    "lin1_parameters_id = list(map(id, lin1_parameters))\n",
    "other_parameters = list(filter(lambda p: id(p) not in lin1_parameters_id,\n",
    "                           all_parameters))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "现在获得了两组参数,一组是linear1,另一组是其他的(本程序中即linear2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear1组参数id: \n",
      "140705409071936\n",
      "140705409072008\n",
      "\n",
      "\n",
      "other组参数id: \n",
      "140705409072296\n",
      "140705409072656\n"
     ]
    }
   ],
   "source": [
    "print('linear1组参数id: ')\n",
    "for p in lin1_parameters:\n",
    "    print(id(p))\n",
    "    \n",
    "print('\\n')\n",
    "print('other组参数id: ')\n",
    "for p in other_parameters:\n",
    "    print(id(p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "构造optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD([\n",
    "                {'params': lin1_parameters},\n",
    "                {'params': other_parameters, 'lr': 1e-3}\n",
    "            ], lr=1e-4)\n",
    "# linear1层的学习率1e-4,其它层1e-3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 719.4832153320312\n",
      "1 618.053466796875\n",
      "2 556.1675415039062\n",
      "3 500.43212890625\n",
      "4 447.5682678222656\n",
      "5 396.4813537597656\n",
      "6 347.07867431640625\n",
      "7 300.09326171875\n",
      "8 256.21868896484375\n",
      "9 216.0878448486328\n",
      "10 180.35525512695312\n",
      "11 149.25791931152344\n",
      "12 122.67479705810547\n",
      "13 100.36481475830078\n",
      "14 81.89598083496094\n",
      "15 66.74868774414062\n",
      "16 54.42697525024414\n",
      "17 44.45567321777344\n",
      "18 36.39970397949219\n",
      "19 29.910730361938477\n",
      "20 24.662458419799805\n",
      "21 20.40622901916504\n",
      "22 16.947093963623047\n",
      "23 14.121529579162598\n",
      "24 11.805665969848633\n",
      "25 9.908512115478516\n",
      "26 8.346722602844238\n",
      "27 7.056419849395752\n",
      "28 5.984086036682129\n",
      "29 5.090843677520752\n",
      "30 4.344158172607422\n",
      "31 3.7168867588043213\n",
      "32 3.1883127689361572\n",
      "33 2.7413690090179443\n",
      "34 2.3627569675445557\n",
      "35 2.0411109924316406\n",
      "36 1.7670531272888184\n",
      "37 1.5327770709991455\n",
      "38 1.3317700624465942\n",
      "39 1.1588612794876099\n",
      "40 1.0100197792053223\n",
      "41 0.8820691108703613\n",
      "42 0.7714465856552124\n",
      "43 0.6757064461708069\n",
      "44 0.592779815196991\n",
      "45 0.5206921696662903\n",
      "46 0.45796385407447815\n",
      "47 0.403344064950943\n",
      "48 0.3557352125644684\n",
      "49 0.31413477659225464\n",
      "50 0.27769696712493896\n",
      "51 0.24573519825935364\n",
      "52 0.2176705002784729\n",
      "53 0.19302386045455933\n",
      "54 0.17133362591266632\n",
      "55 0.15221278369426727\n",
      "56 0.135351300239563\n",
      "57 0.12046048790216446\n",
      "58 0.1073007881641388\n",
      "59 0.09565050154924393\n",
      "60 0.08532743155956268\n",
      "61 0.07617135345935822\n",
      "62 0.06804817914962769\n",
      "63 0.06084805354475975\n",
      "64 0.05446131154894829\n",
      "65 0.04884392023086548\n",
      "66 0.043833404779434204\n",
      "67 0.039361849427223206\n",
      "68 0.03537042811512947\n",
      "69 0.03180677816271782\n",
      "70 0.028617050498723984\n",
      "71 0.025749675929546356\n",
      "72 0.023186029866337776\n",
      "73 0.02088870480656624\n",
      "74 0.018828196451067924\n",
      "75 0.016980471089482307\n",
      "76 0.015320717357099056\n",
      "77 0.013830263167619705\n",
      "78 0.012490017339587212\n",
      "79 0.011284894309937954\n",
      "80 0.01020009908825159\n",
      "81 0.00922376848757267\n",
      "82 0.008343766443431377\n",
      "83 0.007550488226115704\n",
      "84 0.0068353088572621346\n",
      "85 0.006190172396600246\n",
      "86 0.005607489496469498\n",
      "87 0.005081566050648689\n",
      "88 0.00460641598328948\n",
      "89 0.004176917020231485\n",
      "90 0.0037886006757616997\n",
      "91 0.0034373654052615166\n",
      "92 0.0031197601929306984\n",
      "93 0.0028322283178567886\n",
      "94 0.0025717862881720066\n",
      "95 0.002335888333618641\n",
      "96 0.0021221640054136515\n",
      "97 0.0019284778973087668\n",
      "98 0.0017529240576550364\n",
      "99 0.0015937236603349447\n",
      "100 0.0014493277994915843\n",
      "101 0.0013182209804654121\n",
      "102 0.0011992763029411435\n",
      "103 0.0010912807192653418\n",
      "104 0.0009932058164849877\n",
      "105 0.0009040983277373016\n",
      "106 0.0008233404951170087\n",
      "107 0.0007499091443605721\n",
      "108 0.0006831525824964046\n",
      "109 0.0006224379176273942\n",
      "110 0.0005672484403476119\n",
      "111 0.0005170325748622417\n",
      "112 0.0004713317903224379\n",
      "113 0.0004297299892641604\n",
      "114 0.0003918729198630899\n",
      "115 0.0003573991998564452\n",
      "116 0.0003260155499447137\n",
      "117 0.0002974196686409414\n",
      "118 0.0002713669091463089\n",
      "119 0.0002476317167747766\n",
      "120 0.00022600177908316255\n",
      "121 0.0002062939602183178\n",
      "122 0.00018832141358871013\n",
      "123 0.0001719275169307366\n",
      "124 0.0001569933956488967\n",
      "125 0.00014336439198814332\n",
      "126 0.00013093784218654037\n",
      "127 0.0001196042139781639\n",
      "128 0.0001092585880542174\n",
      "129 9.982137999031693e-05\n",
      "130 9.120586764765903e-05\n",
      "131 8.33444792078808e-05\n",
      "132 7.616882066940889e-05\n",
      "133 6.961503822822124e-05\n",
      "134 6.363449210766703e-05\n",
      "135 5.817100827698596e-05\n",
      "136 5.318074545357376e-05\n",
      "137 4.8620247980579734e-05\n",
      "138 4.4461063225753605e-05\n",
      "139 4.0658000216353685e-05\n",
      "140 3.7181245716055855e-05\n",
      "141 3.400376590434462e-05\n",
      "142 3.110080797341652e-05\n",
      "143 2.8448537705116905e-05\n",
      "144 2.6025612896773964e-05\n",
      "145 2.3810694983694702e-05\n",
      "146 2.1784513592137955e-05\n",
      "147 1.9931732822442427e-05\n",
      "148 1.8238761185784824e-05\n",
      "149 1.669217635935638e-05\n",
      "150 1.5273904864443466e-05\n",
      "151 1.3978798961034045e-05\n",
      "152 1.279618481930811e-05\n",
      "153 1.1711815204762388e-05\n",
      "154 1.071973474608967e-05\n",
      "155 9.814746590564027e-06\n",
      "156 8.985691238194704e-06\n",
      "157 8.226681529777125e-06\n",
      "158 7.533013558713719e-06\n",
      "159 6.897260846017161e-06\n",
      "160 6.3151273934636265e-06\n",
      "161 5.784675977338338e-06\n",
      "162 5.296439212543191e-06\n",
      "163 4.851282938034274e-06\n",
      "164 4.442661975190276e-06\n",
      "165 4.068862381245708e-06\n",
      "166 3.7270233406161424e-06\n",
      "167 3.413488684600452e-06\n",
      "168 3.1275621950044297e-06\n",
      "169 2.8650440526689636e-06\n",
      "170 2.624645276227966e-06\n",
      "171 2.405057784926612e-06\n",
      "172 2.2035642359696794e-06\n",
      "173 2.019183966694982e-06\n",
      "174 1.850062517405604e-06\n",
      "175 1.6953827071120031e-06\n",
      "176 1.5531543340330245e-06\n",
      "177 1.4232090279620024e-06\n",
      "178 1.3044416391494451e-06\n",
      "179 1.1956101388932439e-06\n",
      "180 1.0957942322420422e-06\n",
      "181 1.0046904890259611e-06\n",
      "182 9.207004154632159e-07\n",
      "183 8.440935062026256e-07\n",
      "184 7.73405361087498e-07\n",
      "185 7.091608722475939e-07\n",
      "186 6.500075642179581e-07\n",
      "187 5.959356030871277e-07\n",
      "188 5.460437932924833e-07\n",
      "189 5.007885874874773e-07\n",
      "190 4.5921461833131616e-07\n",
      "191 4.2098395169887226e-07\n",
      "192 3.8619594988631434e-07\n",
      "193 3.539971658028662e-07\n",
      "194 3.244428512516606e-07\n",
      "195 2.975311303998751e-07\n",
      "196 2.726736170188815e-07\n",
      "197 2.5038809781108284e-07\n",
      "198 2.2951981293317658e-07\n",
      "199 2.1044718323537381e-07\n",
      "200 1.931230571017295e-07\n",
      "201 1.7704486765524052e-07\n",
      "202 1.625433725394032e-07\n",
      "203 1.4899816846991598e-07\n",
      "204 1.3666438292148086e-07\n",
      "205 1.2532166238088394e-07\n",
      "206 1.1494875451489861e-07\n",
      "207 1.054767295727288e-07\n",
      "208 9.676008971837291e-08\n",
      "209 8.872893886291422e-08\n",
      "210 8.135914697504631e-08\n",
      "211 7.470005414234038e-08\n",
      "212 6.852687306491134e-08\n",
      "213 6.293034005011577e-08\n",
      "214 5.7790256136058815e-08\n",
      "215 5.302708672161316e-08\n",
      "216 4.8682117892440147e-08\n",
      "217 4.453647051150256e-08\n",
      "218 4.099248585021087e-08\n",
      "219 3.761395817036828e-08\n",
      "220 3.453242669593237e-08\n",
      "221 3.170368501059784e-08\n",
      "222 2.915122720992258e-08\n",
      "223 2.6717120960029206e-08\n",
      "224 2.451883673870725e-08\n",
      "225 2.2580659120308155e-08\n",
      "226 2.0710798409595554e-08\n",
      "227 1.9040477639009623e-08\n",
      "228 1.7526446072224644e-08\n",
      "229 1.6122124080197864e-08\n",
      "230 1.4806063042271944e-08\n",
      "231 1.3633435713700237e-08\n",
      "232 1.2587334730085331e-08\n",
      "233 1.1583493275679757e-08\n",
      "234 1.0632591695980409e-08\n",
      "235 9.816670143436568e-09\n",
      "236 9.059893280038978e-09\n",
      "237 8.38562641547469e-09\n",
      "238 7.752122499482539e-09\n",
      "239 7.144894009769587e-09\n",
      "240 6.591914125664289e-09\n",
      "241 6.152251152968802e-09\n",
      "242 5.6817590632363135e-09\n",
      "243 5.275397008119853e-09\n",
      "244 4.890560845183245e-09\n",
      "245 4.5295536210687715e-09\n",
      "246 4.211996085246028e-09\n",
      "247 3.908982026956664e-09\n",
      "248 3.6504110845214655e-09\n",
      "249 3.4168203821849374e-09\n",
      "250 3.187831110196271e-09\n",
      "251 2.98542235377397e-09\n",
      "252 2.8059994328089033e-09\n",
      "253 2.626949102690901e-09\n",
      "254 2.4697992540012592e-09\n",
      "255 2.3114088421039014e-09\n",
      "256 2.1845512065965522e-09\n",
      "257 2.0542922918309614e-09\n",
      "258 1.9309427390368228e-09\n",
      "259 1.8381356436947272e-09\n",
      "260 1.7248911188261218e-09\n",
      "261 1.6398804536521538e-09\n",
      "262 1.5482324311477669e-09\n",
      "263 1.4635379574912122e-09\n",
      "264 1.3910612661760524e-09\n",
      "265 1.317505660125562e-09\n",
      "266 1.2539014271339965e-09\n",
      "267 1.2011907024600532e-09\n",
      "268 1.1445275838184443e-09\n",
      "269 1.0953595808160799e-09\n",
      "270 1.0399190397691882e-09\n",
      "271 1.0023235574863065e-09\n",
      "272 9.588654314995892e-10\n",
      "273 9.140234680238279e-10\n",
      "274 8.714531318787522e-10\n",
      "275 8.341509150078252e-10\n",
      "276 8.028351317079796e-10\n",
      "277 7.760427300773642e-10\n",
      "278 7.34666716351029e-10\n",
      "279 7.090298348444435e-10\n",
      "280 6.824536491478739e-10\n",
      "281 6.50852871597607e-10\n",
      "282 6.332582236368012e-10\n",
      "283 6.151034126489208e-10\n",
      "284 5.901392152729557e-10\n",
      "285 5.691445092992353e-10\n",
      "286 5.507771461132904e-10\n",
      "287 5.32691946109054e-10\n",
      "288 5.124157764768711e-10\n",
      "289 4.940373665718312e-10\n",
      "290 4.766944616818591e-10\n",
      "291 4.6186365842970645e-10\n",
      "292 4.470791514776806e-10\n",
      "293 4.3581016573313036e-10\n",
      "294 4.2106490516502504e-10\n",
      "295 4.066213477038616e-10\n",
      "296 3.981817098264173e-10\n",
      "297 3.8970893179168797e-10\n",
      "298 3.72095521061766e-10\n",
      "299 3.6663283520255163e-10\n",
      "300 3.573504825382656e-10\n",
      "301 3.487677924240984e-10\n",
      "302 3.3633168472491093e-10\n",
      "303 3.290345773621084e-10\n",
      "304 3.224511213595349e-10\n",
      "305 3.122619940398863e-10\n",
      "306 3.0483571222816863e-10\n",
      "307 2.9414332081145744e-10\n",
      "308 2.8667790363812173e-10\n",
      "309 2.8062335788447967e-10\n",
      "310 2.762368667141857e-10\n",
      "311 2.6522151141961103e-10\n",
      "312 2.57844356976733e-10\n",
      "313 2.5349577992273e-10\n",
      "314 2.478507399317209e-10\n",
      "315 2.41882402995941e-10\n",
      "316 2.3629426193494396e-10\n",
      "317 2.3070964583205011e-10\n",
      "318 2.264485127190241e-10\n",
      "319 2.2164882429454025e-10\n",
      "320 2.1533744232193897e-10\n",
      "321 2.097950702051321e-10\n",
      "322 2.064158427517171e-10\n",
      "323 1.990261011552974e-10\n",
      "324 1.9497652103961371e-10\n",
      "325 1.9140954099494678e-10\n",
      "326 1.8693983860895713e-10\n",
      "327 1.8207133023473432e-10\n",
      "328 1.783010128431073e-10\n",
      "329 1.763045681668629e-10\n",
      "330 1.7145321273837055e-10\n",
      "331 1.6931771262829187e-10\n",
      "332 1.6673835923075586e-10\n",
      "333 1.6416644432748484e-10\n",
      "334 1.6168200112076647e-10\n",
      "335 1.5868138747432425e-10\n",
      "336 1.5622357574240908e-10\n",
      "337 1.5223147742382537e-10\n",
      "338 1.4958895233618819e-10\n",
      "339 1.4682752236261365e-10\n",
      "340 1.436487179207191e-10\n",
      "341 1.4157740257925155e-10\n",
      "342 1.410829092440835e-10\n",
      "343 1.380528469319131e-10\n",
      "344 1.3580145341585137e-10\n",
      "345 1.341790289988154e-10\n",
      "346 1.3181483682345174e-10\n",
      "347 1.3028327028319353e-10\n",
      "348 1.2861381404327688e-10\n",
      "349 1.2620478273550617e-10\n",
      "350 1.2542905603041277e-10\n",
      "351 1.238803642999997e-10\n",
      "352 1.218108947043106e-10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "353 1.2094443502252972e-10\n",
      "354 1.1777642749954964e-10\n",
      "355 1.1760770135538223e-10\n",
      "356 1.1544241951266798e-10\n",
      "357 1.1402352673162142e-10\n",
      "358 1.1242246023002167e-10\n",
      "359 1.1167516911214648e-10\n",
      "360 1.0990484911044263e-10\n",
      "361 1.0966394459188678e-10\n",
      "362 1.0963921437401325e-10\n",
      "363 1.0603967703914918e-10\n",
      "364 1.03834024711702e-10\n",
      "365 1.039718450224214e-10\n",
      "366 1.0166703590108739e-10\n",
      "367 1.0114989401621699e-10\n",
      "368 9.834176528666916e-11\n",
      "369 9.60351104195567e-11\n",
      "370 9.521219923591673e-11\n",
      "371 9.493408836824813e-11\n",
      "372 9.308730175572322e-11\n",
      "373 9.198990180703248e-11\n",
      "374 9.021075553228286e-11\n",
      "375 8.961704989207675e-11\n",
      "376 8.807096718577156e-11\n",
      "377 8.61079402225684e-11\n",
      "378 8.594966405262028e-11\n",
      "379 8.452996635988086e-11\n",
      "380 8.392590788997012e-11\n",
      "381 8.171358034658738e-11\n",
      "382 8.029973908030286e-11\n",
      "383 7.93237697749305e-11\n",
      "384 7.853767636234465e-11\n",
      "385 7.892052289459883e-11\n",
      "386 7.902740961629462e-11\n",
      "387 7.743728575038133e-11\n",
      "388 7.645977601056231e-11\n",
      "389 7.571010485207808e-11\n",
      "390 7.520549460959813e-11\n",
      "391 7.54181994633285e-11\n",
      "392 7.461733314562125e-11\n",
      "393 7.3604816686057e-11\n",
      "394 7.3249614707116e-11\n",
      "395 7.211352348601707e-11\n",
      "396 7.178711791677728e-11\n",
      "397 7.167869769952873e-11\n",
      "398 7.025070108968023e-11\n",
      "399 6.825126575016327e-11\n",
      "400 6.793333950927405e-11\n",
      "401 6.772558902579107e-11\n",
      "402 6.730412061006774e-11\n",
      "403 6.625886644906487e-11\n",
      "404 6.461721435702117e-11\n",
      "405 6.47099734907286e-11\n",
      "406 6.391782242376465e-11\n",
      "407 6.264266882993752e-11\n",
      "408 6.173965505507084e-11\n",
      "409 6.116362971653189e-11\n",
      "410 6.077430919626536e-11\n",
      "411 6.030231869402769e-11\n",
      "412 5.947273923334606e-11\n",
      "413 5.867167862660949e-11\n",
      "414 5.855109452834739e-11\n",
      "415 5.7634869099487673e-11\n",
      "416 5.798097418852066e-11\n",
      "417 5.714314438298729e-11\n",
      "418 5.7525491314880384e-11\n",
      "419 5.753773499317383e-11\n",
      "420 5.72689846933816e-11\n",
      "421 5.631775948367057e-11\n",
      "422 5.598362051717487e-11\n",
      "423 5.5867376697049664e-11\n",
      "424 5.629419153052595e-11\n",
      "425 5.5520841396594633e-11\n",
      "426 5.46322084793438e-11\n",
      "427 5.4010414196614676e-11\n",
      "428 5.341607364761636e-11\n",
      "429 5.3519549902958374e-11\n",
      "430 5.247411186126705e-11\n",
      "431 5.307565845158457e-11\n",
      "432 5.2371048470112314e-11\n",
      "433 5.2449510012930745e-11\n",
      "434 5.1879195384074706e-11\n",
      "435 5.1574945703070085e-11\n",
      "436 5.1512395043973314e-11\n",
      "437 5.08322897663227e-11\n",
      "438 5.057060326052465e-11\n",
      "439 4.9787240302689995e-11\n",
      "440 4.823476340565236e-11\n",
      "441 4.8459992962879284e-11\n",
      "442 4.797246627719076e-11\n",
      "443 4.824013410953398e-11\n",
      "444 4.7559997606860804e-11\n",
      "445 4.690832444698145e-11\n",
      "446 4.619358368040949e-11\n",
      "447 4.583974866356755e-11\n",
      "448 4.605983650041168e-11\n",
      "449 4.5755635391664384e-11\n",
      "450 4.547212259509159e-11\n",
      "451 4.5017298916372184e-11\n",
      "452 4.5089352390670356e-11\n",
      "453 4.437321690642371e-11\n",
      "454 4.417576374149412e-11\n",
      "455 4.4186581477090314e-11\n",
      "456 4.324371069563959e-11\n",
      "457 4.303992579002269e-11\n",
      "458 4.266882333570088e-11\n",
      "459 4.2662408328286716e-11\n",
      "460 4.2061472360632735e-11\n",
      "461 4.153392560435343e-11\n",
      "462 4.1709347781138106e-11\n",
      "463 4.15860852698291e-11\n",
      "464 4.1531614952683427e-11\n",
      "465 4.106182407981329e-11\n",
      "466 4.0790346794716825e-11\n",
      "467 4.062897934753451e-11\n",
      "468 3.990976646384148e-11\n",
      "469 4.011613263799063e-11\n",
      "470 3.9666533946380866e-11\n",
      "471 3.945664281412853e-11\n",
      "472 3.835248785222234e-11\n",
      "473 3.8230120458226935e-11\n",
      "474 3.747497104300557e-11\n",
      "475 3.7175280215295814e-11\n",
      "476 3.713800100779707e-11\n",
      "477 3.680154792018442e-11\n",
      "478 3.743382687160235e-11\n",
      "479 3.680740781608627e-11\n",
      "480 3.6479482629081517e-11\n",
      "481 3.598047901287593e-11\n",
      "482 3.572271645158054e-11\n",
      "483 3.5526505348659754e-11\n",
      "484 3.541624979397362e-11\n",
      "485 3.4761606787503396e-11\n",
      "486 3.4769877949036854e-11\n",
      "487 3.460022199308632e-11\n",
      "488 3.453215144388899e-11\n",
      "489 3.4166957457726355e-11\n",
      "490 3.428968220475781e-11\n",
      "491 3.439980245101282e-11\n",
      "492 3.4364174700263206e-11\n",
      "493 3.4181008717881767e-11\n",
      "494 3.368537393466653e-11\n",
      "495 3.368653272994848e-11\n",
      "496 3.291827227469568e-11\n",
      "497 3.2805473615393765e-11\n",
      "498 3.3228246543171025e-11\n",
      "499 3.3127479925898484e-11\n"
     ]
    }
   ],
   "source": [
    "for t in range(500):\n",
    "    # Forward pass: Compute predicted y by passing x to the model\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # Compute and print loss\n",
    "    loss = criterion(y_pred, y)\n",
    "    print(t, loss.item())\n",
    "\n",
    "    # Zero gradients, perform a backward pass, and update the weights.\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
 }
	{
	"cells": [
	{
	"cell_type": "markdown",
	"metadata": {},
	"source": [
	"分层设置学习率"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 4,
	"metadata": {},
	"outputs": [],
	"source": [
	"import torch\n",
	"import torch.nn as nn\n",
	"import torch.nn.functional as F\n",
	"import torch.optim as optim"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 3,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"0.4.0\n"
	]
	}
	],
	"source": [
	"print(torch.__version__)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 5,
	"metadata": {},
	"outputs": [],
	"source": [
	"# 构建一个简单多层网络结构\n",
	"class TwoLayerNet(torch.nn.Module):\n",
	" def __init__(self, D_in, H, D_out):\n",
	" \"\"\"\n",
	" In the constructor we instantiate two nn.Linear modules and assign them as\n",
	" member variables.\n",
	" \"\"\"\n",
	" super(TwoLayerNet, self).__init__()\n",
	" self.linear1 = torch.nn.Linear(D_in, H)\n",
	" self.linear2 = torch.nn.Linear(H, D_out)\n",
	"\n",
	" def forward(self, x):\n",
	" \"\"\"\n",
	" In the forward function we accept a Tensor of input data and we must return\n",
	" a Tensor of output data. We can use Modules defined in the constructor as\n",
	" well as arbitrary operators on Tensors.\n",
	" \"\"\"\n",
	" h_relu = F.relu(self.linear1(x))\n",
	" y_pred = self.linear2(h_relu)\n",
	" return y_pred"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 7,
	"metadata": {},
	"outputs": [],
	"source": [
	"# N is batch size; D_in is input dimension;\n",
	"# H is hidden dimension; D_out is output dimension.\n",
	"N, D_in, H, D_out = 64, 1000, 100, 10"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 8,
	"metadata": {},
	"outputs": [],
	"source": [
	"x = torch.randn(N, D_in)\n",
	"y = torch.randn(N, D_out)\n",
	"\n",
	"# Construct our model by instantiating the class defined above\n",
	"model = TwoLayerNet(D_in, H, D_out)\n",
	"\n",
	"# Construct our loss function and an Optimizer. The call to model.parameters()\n",
	"# in the SGD constructor will contain the learnable parameters of the two\n",
	"# nn.Linear modules which are members of the model.\n",
	"criterion = torch.nn.MSELoss(size_average=False)"
	]
	},
	{
	"cell_type": "markdown",
	"metadata": {},
	"source": [
	"查看模型的参数名称"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 20,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"参数名: linear1.weight , id: 140705409071936\n",
	"参数名: linear1.bias , id: 140705409072008\n",
	"参数名: linear2.weight , id: 140705409072296\n",
	"参数名: linear2.bias , id: 140705409072656\n"
	]
	}
	],
	"source": [
	"for pname, p in model.named_parameters():\n",
	" print('参数名: {: <18}, id: {}'.format(pname, id(p)))"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 21,
	"metadata": {},
	"outputs": [],
	"source": [
	"all_parameters = model.parameters()\n",
	"\n",
	"lin1_parameters = []\n",
	"for pname, p in model.named_parameters():\n",
	" if pname.find('linear1') >= 0:\n",
	" lin1_parameters.append(p)\n",
	"\n",
	"lin1_parameters_id = list(map(id, lin1_parameters))\n",
	"other_parameters = list(filter(lambda p: id(p) not in lin1_parameters_id,\n",
	" all_parameters))"
	]
	},
	{
	"cell_type": "markdown",
	"metadata": {},
	"source": [
	"现在获得了两组参数,一组是linear1,另一组是其他的(本程序中即linear2)"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 23,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"linear1组参数id: \n",
	"140705409071936\n",
	"140705409072008\n",
	"\n",
	"\n",
	"other组参数id: \n",
	"140705409072296\n",
	"140705409072656\n"
	]
	}
	],
	"source": [
	"print('linear1组参数id: ')\n",
	"for p in lin1_parameters:\n",
	" print(id(p))\n",
	" \n",
	"print('\\n')\n",
	"print('other组参数id: ')\n",
	"for p in other_parameters:\n",
	" print(id(p))"
	]
	},
	{
	"cell_type": "markdown",
	"metadata": {},
	"source": [
	"构造optim"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 24,
	"metadata": {},
	"outputs": [],
	"source": [
	"optimizer = optim.SGD([\n",
	" {'params': lin1_parameters},\n",
	" {'params': other_parameters, 'lr': 1e-3}\n",
	" ], lr=1e-4)\n",
	"# linear1层的学习率1e-4,其它层1e-3"
	]
	},
	{
	"cell_type": "markdown",
	"metadata": {},
	"source": [
	"训练网络"
	]
	},
	{
	"cell_type": "code",
	"execution_count": 25,
	"metadata": {},
	"outputs": [
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"0 719.4832153320312\n",
	"1 618.053466796875\n",
	"2 556.1675415039062\n",
	"3 500.43212890625\n",
	"4 447.5682678222656\n",
	"5 396.4813537597656\n",
	"6 347.07867431640625\n",
	"7 300.09326171875\n",
	"8 256.21868896484375\n",
	"9 216.0878448486328\n",
	"10 180.35525512695312\n",
	"11 149.25791931152344\n",
	"12 122.67479705810547\n",
	"13 100.36481475830078\n",
	"14 81.89598083496094\n",
	"15 66.74868774414062\n",
	"16 54.42697525024414\n",
	"17 44.45567321777344\n",
	"18 36.39970397949219\n",
	"19 29.910730361938477\n",
	"20 24.662458419799805\n",
	"21 20.40622901916504\n",
	"22 16.947093963623047\n",
	"23 14.121529579162598\n",
	"24 11.805665969848633\n",
	"25 9.908512115478516\n",
	"26 8.346722602844238\n",
	"27 7.056419849395752\n",
	"28 5.984086036682129\n",
	"29 5.090843677520752\n",
	"30 4.344158172607422\n",
	"31 3.7168867588043213\n",
	"32 3.1883127689361572\n",
	"33 2.7413690090179443\n",
	"34 2.3627569675445557\n",
	"35 2.0411109924316406\n",
	"36 1.7670531272888184\n",
	"37 1.5327770709991455\n",
	"38 1.3317700624465942\n",
	"39 1.1588612794876099\n",
	"40 1.0100197792053223\n",
	"41 0.8820691108703613\n",
	"42 0.7714465856552124\n",
	"43 0.6757064461708069\n",
	"44 0.592779815196991\n",
	"45 0.5206921696662903\n",
	"46 0.45796385407447815\n",
	"47 0.403344064950943\n",
	"48 0.3557352125644684\n",
	"49 0.31413477659225464\n",
	"50 0.27769696712493896\n",
	"51 0.24573519825935364\n",
	"52 0.2176705002784729\n",
	"53 0.19302386045455933\n",
	"54 0.17133362591266632\n",
	"55 0.15221278369426727\n",
	"56 0.135351300239563\n",
	"57 0.12046048790216446\n",
	"58 0.1073007881641388\n",
	"59 0.09565050154924393\n",
	"60 0.08532743155956268\n",
	"61 0.07617135345935822\n",
	"62 0.06804817914962769\n",
	"63 0.06084805354475975\n",
	"64 0.05446131154894829\n",
	"65 0.04884392023086548\n",
	"66 0.043833404779434204\n",
	"67 0.039361849427223206\n",
	"68 0.03537042811512947\n",
	"69 0.03180677816271782\n",
	"70 0.028617050498723984\n",
	"71 0.025749675929546356\n",
	"72 0.023186029866337776\n",
	"73 0.02088870480656624\n",
	"74 0.018828196451067924\n",
	"75 0.016980471089482307\n",
	"76 0.015320717357099056\n",
	"77 0.013830263167619705\n",
	"78 0.012490017339587212\n",
	"79 0.011284894309937954\n",
	"80 0.01020009908825159\n",
	"81 0.00922376848757267\n",
	"82 0.008343766443431377\n",
	"83 0.007550488226115704\n",
	"84 0.0068353088572621346\n",
	"85 0.006190172396600246\n",
	"86 0.005607489496469498\n",
	"87 0.005081566050648689\n",
	"88 0.00460641598328948\n",
	"89 0.004176917020231485\n",
	"90 0.0037886006757616997\n",
	"91 0.0034373654052615166\n",
	"92 0.0031197601929306984\n",
	"93 0.0028322283178567886\n",
	"94 0.0025717862881720066\n",
	"95 0.002335888333618641\n",
	"96 0.0021221640054136515\n",
	"97 0.0019284778973087668\n",
	"98 0.0017529240576550364\n",
	"99 0.0015937236603349447\n",
	"100 0.0014493277994915843\n",
	"101 0.0013182209804654121\n",
	"102 0.0011992763029411435\n",
	"103 0.0010912807192653418\n",
	"104 0.0009932058164849877\n",
	"105 0.0009040983277373016\n",
	"106 0.0008233404951170087\n",
	"107 0.0007499091443605721\n",
	"108 0.0006831525824964046\n",
	"109 0.0006224379176273942\n",
	"110 0.0005672484403476119\n",
	"111 0.0005170325748622417\n",
	"112 0.0004713317903224379\n",
	"113 0.0004297299892641604\n",
	"114 0.0003918729198630899\n",
	"115 0.0003573991998564452\n",
	"116 0.0003260155499447137\n",
	"117 0.0002974196686409414\n",
	"118 0.0002713669091463089\n",
	"119 0.0002476317167747766\n",
	"120 0.00022600177908316255\n",
	"121 0.0002062939602183178\n",
	"122 0.00018832141358871013\n",
	"123 0.0001719275169307366\n",
	"124 0.0001569933956488967\n",
	"125 0.00014336439198814332\n",
	"126 0.00013093784218654037\n",
	"127 0.0001196042139781639\n",
	"128 0.0001092585880542174\n",
	"129 9.982137999031693e-05\n",
	"130 9.120586764765903e-05\n",
	"131 8.33444792078808e-05\n",
	"132 7.616882066940889e-05\n",
	"133 6.961503822822124e-05\n",
	"134 6.363449210766703e-05\n",
	"135 5.817100827698596e-05\n",
	"136 5.318074545357376e-05\n",
	"137 4.8620247980579734e-05\n",
	"138 4.4461063225753605e-05\n",
	"139 4.0658000216353685e-05\n",
	"140 3.7181245716055855e-05\n",
	"141 3.400376590434462e-05\n",
	"142 3.110080797341652e-05\n",
	"143 2.8448537705116905e-05\n",
	"144 2.6025612896773964e-05\n",
	"145 2.3810694983694702e-05\n",
	"146 2.1784513592137955e-05\n",
	"147 1.9931732822442427e-05\n",
	"148 1.8238761185784824e-05\n",
	"149 1.669217635935638e-05\n",
	"150 1.5273904864443466e-05\n",
	"151 1.3978798961034045e-05\n",
	"152 1.279618481930811e-05\n",
	"153 1.1711815204762388e-05\n",
	"154 1.071973474608967e-05\n",
	"155 9.814746590564027e-06\n",
	"156 8.985691238194704e-06\n",
	"157 8.226681529777125e-06\n",
	"158 7.533013558713719e-06\n",
	"159 6.897260846017161e-06\n",
	"160 6.3151273934636265e-06\n",
	"161 5.784675977338338e-06\n",
	"162 5.296439212543191e-06\n",
	"163 4.851282938034274e-06\n",
	"164 4.442661975190276e-06\n",
	"165 4.068862381245708e-06\n",
	"166 3.7270233406161424e-06\n",
	"167 3.413488684600452e-06\n",
	"168 3.1275621950044297e-06\n",
	"169 2.8650440526689636e-06\n",
	"170 2.624645276227966e-06\n",
	"171 2.405057784926612e-06\n",
	"172 2.2035642359696794e-06\n",
	"173 2.019183966694982e-06\n",
	"174 1.850062517405604e-06\n",
	"175 1.6953827071120031e-06\n",
	"176 1.5531543340330245e-06\n",
	"177 1.4232090279620024e-06\n",
	"178 1.3044416391494451e-06\n",
	"179 1.1956101388932439e-06\n",
	"180 1.0957942322420422e-06\n",
	"181 1.0046904890259611e-06\n",
	"182 9.207004154632159e-07\n",
	"183 8.440935062026256e-07\n",
	"184 7.73405361087498e-07\n",
	"185 7.091608722475939e-07\n",
	"186 6.500075642179581e-07\n",
	"187 5.959356030871277e-07\n",
	"188 5.460437932924833e-07\n",
	"189 5.007885874874773e-07\n",
	"190 4.5921461833131616e-07\n",
	"191 4.2098395169887226e-07\n",
	"192 3.8619594988631434e-07\n",
	"193 3.539971658028662e-07\n",
	"194 3.244428512516606e-07\n",
	"195 2.975311303998751e-07\n",
	"196 2.726736170188815e-07\n",
	"197 2.5038809781108284e-07\n",
	"198 2.2951981293317658e-07\n",
	"199 2.1044718323537381e-07\n",
	"200 1.931230571017295e-07\n",
	"201 1.7704486765524052e-07\n",
	"202 1.625433725394032e-07\n",
	"203 1.4899816846991598e-07\n",
	"204 1.3666438292148086e-07\n",
	"205 1.2532166238088394e-07\n",
	"206 1.1494875451489861e-07\n",
	"207 1.054767295727288e-07\n",
	"208 9.676008971837291e-08\n",
	"209 8.872893886291422e-08\n",
	"210 8.135914697504631e-08\n",
	"211 7.470005414234038e-08\n",
	"212 6.852687306491134e-08\n",
	"213 6.293034005011577e-08\n",
	"214 5.7790256136058815e-08\n",
	"215 5.302708672161316e-08\n",
	"216 4.8682117892440147e-08\n",
	"217 4.453647051150256e-08\n",
	"218 4.099248585021087e-08\n",
	"219 3.761395817036828e-08\n",
	"220 3.453242669593237e-08\n",
	"221 3.170368501059784e-08\n",
	"222 2.915122720992258e-08\n",
	"223 2.6717120960029206e-08\n",
	"224 2.451883673870725e-08\n",
	"225 2.2580659120308155e-08\n",
	"226 2.0710798409595554e-08\n",
	"227 1.9040477639009623e-08\n",
	"228 1.7526446072224644e-08\n",
	"229 1.6122124080197864e-08\n",
	"230 1.4806063042271944e-08\n",
	"231 1.3633435713700237e-08\n",
	"232 1.2587334730085331e-08\n",
	"233 1.1583493275679757e-08\n",
	"234 1.0632591695980409e-08\n",
	"235 9.816670143436568e-09\n",
	"236 9.059893280038978e-09\n",
	"237 8.38562641547469e-09\n",
	"238 7.752122499482539e-09\n",
	"239 7.144894009769587e-09\n",
	"240 6.591914125664289e-09\n",
	"241 6.152251152968802e-09\n",
	"242 5.6817590632363135e-09\n",
	"243 5.275397008119853e-09\n",
	"244 4.890560845183245e-09\n",
	"245 4.5295536210687715e-09\n",
	"246 4.211996085246028e-09\n",
	"247 3.908982026956664e-09\n",
	"248 3.6504110845214655e-09\n",
	"249 3.4168203821849374e-09\n",
	"250 3.187831110196271e-09\n",
	"251 2.98542235377397e-09\n",
	"252 2.8059994328089033e-09\n",
	"253 2.626949102690901e-09\n",
	"254 2.4697992540012592e-09\n",
	"255 2.3114088421039014e-09\n",
	"256 2.1845512065965522e-09\n",
	"257 2.0542922918309614e-09\n",
	"258 1.9309427390368228e-09\n",
	"259 1.8381356436947272e-09\n",
	"260 1.7248911188261218e-09\n",
	"261 1.6398804536521538e-09\n",
	"262 1.5482324311477669e-09\n",
	"263 1.4635379574912122e-09\n",
	"264 1.3910612661760524e-09\n",
	"265 1.317505660125562e-09\n",
	"266 1.2539014271339965e-09\n",
	"267 1.2011907024600532e-09\n",
	"268 1.1445275838184443e-09\n",
	"269 1.0953595808160799e-09\n",
	"270 1.0399190397691882e-09\n",
	"271 1.0023235574863065e-09\n",
	"272 9.588654314995892e-10\n",
	"273 9.140234680238279e-10\n",
	"274 8.714531318787522e-10\n",
	"275 8.341509150078252e-10\n",
	"276 8.028351317079796e-10\n",
	"277 7.760427300773642e-10\n",
	"278 7.34666716351029e-10\n",
	"279 7.090298348444435e-10\n",
	"280 6.824536491478739e-10\n",
	"281 6.50852871597607e-10\n",
	"282 6.332582236368012e-10\n",
	"283 6.151034126489208e-10\n",
	"284 5.901392152729557e-10\n",
	"285 5.691445092992353e-10\n",
	"286 5.507771461132904e-10\n",
	"287 5.32691946109054e-10\n",
	"288 5.124157764768711e-10\n",
	"289 4.940373665718312e-10\n",
	"290 4.766944616818591e-10\n",
	"291 4.6186365842970645e-10\n",
	"292 4.470791514776806e-10\n",
	"293 4.3581016573313036e-10\n",
	"294 4.2106490516502504e-10\n",
	"295 4.066213477038616e-10\n",
	"296 3.981817098264173e-10\n",
	"297 3.8970893179168797e-10\n",
	"298 3.72095521061766e-10\n",
	"299 3.6663283520255163e-10\n",
	"300 3.573504825382656e-10\n",
	"301 3.487677924240984e-10\n",
	"302 3.3633168472491093e-10\n",
	"303 3.290345773621084e-10\n",
	"304 3.224511213595349e-10\n",
	"305 3.122619940398863e-10\n",
	"306 3.0483571222816863e-10\n",
	"307 2.9414332081145744e-10\n",
	"308 2.8667790363812173e-10\n",
	"309 2.8062335788447967e-10\n",
	"310 2.762368667141857e-10\n",
	"311 2.6522151141961103e-10\n",
	"312 2.57844356976733e-10\n",
	"313 2.5349577992273e-10\n",
	"314 2.478507399317209e-10\n",
	"315 2.41882402995941e-10\n",
	"316 2.3629426193494396e-10\n",
	"317 2.3070964583205011e-10\n",
	"318 2.264485127190241e-10\n",
	"319 2.2164882429454025e-10\n",
	"320 2.1533744232193897e-10\n",
	"321 2.097950702051321e-10\n",
	"322 2.064158427517171e-10\n",
	"323 1.990261011552974e-10\n",
	"324 1.9497652103961371e-10\n",
	"325 1.9140954099494678e-10\n",
	"326 1.8693983860895713e-10\n",
	"327 1.8207133023473432e-10\n",
	"328 1.783010128431073e-10\n",
	"329 1.763045681668629e-10\n",
	"330 1.7145321273837055e-10\n",
	"331 1.6931771262829187e-10\n",
	"332 1.6673835923075586e-10\n",
	"333 1.6416644432748484e-10\n",
	"334 1.6168200112076647e-10\n",
	"335 1.5868138747432425e-10\n",
	"336 1.5622357574240908e-10\n",
	"337 1.5223147742382537e-10\n",
	"338 1.4958895233618819e-10\n",
	"339 1.4682752236261365e-10\n",
	"340 1.436487179207191e-10\n",
	"341 1.4157740257925155e-10\n",
	"342 1.410829092440835e-10\n",
	"343 1.380528469319131e-10\n",
	"344 1.3580145341585137e-10\n",
	"345 1.341790289988154e-10\n",
	"346 1.3181483682345174e-10\n",
	"347 1.3028327028319353e-10\n",
	"348 1.2861381404327688e-10\n",
	"349 1.2620478273550617e-10\n",
	"350 1.2542905603041277e-10\n",
	"351 1.238803642999997e-10\n",
	"352 1.218108947043106e-10\n"
	]
	},
	{
	"name": "stdout",
	"output_type": "stream",
	"text": [
	"353 1.2094443502252972e-10\n",
	"354 1.1777642749954964e-10\n",
	"355 1.1760770135538223e-10\n",
	"356 1.1544241951266798e-10\n",
	"357 1.1402352673162142e-10\n",
	"358 1.1242246023002167e-10\n",
	"359 1.1167516911214648e-10\n",
	"360 1.0990484911044263e-10\n",
	"361 1.0966394459188678e-10\n",
	"362 1.0963921437401325e-10\n",
	"363 1.0603967703914918e-10\n",
	"364 1.03834024711702e-10\n",
	"365 1.039718450224214e-10\n",
	"366 1.0166703590108739e-10\n",
	"367 1.0114989401621699e-10\n",
	"368 9.834176528666916e-11\n",
	"369 9.60351104195567e-11\n",
	"370 9.521219923591673e-11\n",
	"371 9.493408836824813e-11\n",
	"372 9.308730175572322e-11\n",
	"373 9.198990180703248e-11\n",
	"374 9.021075553228286e-11\n",
	"375 8.961704989207675e-11\n",
	"376 8.807096718577156e-11\n",
	"377 8.61079402225684e-11\n",
	"378 8.594966405262028e-11\n",
	"379 8.452996635988086e-11\n",
	"380 8.392590788997012e-11\n",
	"381 8.171358034658738e-11\n",
	"382 8.029973908030286e-11\n",
	"383 7.93237697749305e-11\n",
	"384 7.853767636234465e-11\n",
	"385 7.892052289459883e-11\n",
	"386 7.902740961629462e-11\n",
	"387 7.743728575038133e-11\n",
	"388 7.645977601056231e-11\n",
	"389 7.571010485207808e-11\n",
	"390 7.520549460959813e-11\n",
	"391 7.54181994633285e-11\n",
	"392 7.461733314562125e-11\n",
	"393 7.3604816686057e-11\n",
	"394 7.3249614707116e-11\n",
	"395 7.211352348601707e-11\n",
	"396 7.178711791677728e-11\n",
	"397 7.167869769952873e-11\n",
	"398 7.025070108968023e-11\n",
	"399 6.825126575016327e-11\n",
	"400 6.793333950927405e-11\n",
	"401 6.772558902579107e-11\n",
	"402 6.730412061006774e-11\n",
	"403 6.625886644906487e-11\n",
	"404 6.461721435702117e-11\n",
	"405 6.47099734907286e-11\n",
	"406 6.391782242376465e-11\n",
	"407 6.264266882993752e-11\n",
	"408 6.173965505507084e-11\n",
	"409 6.116362971653189e-11\n",
	"410 6.077430919626536e-11\n",
	"411 6.030231869402769e-11\n",
	"412 5.947273923334606e-11\n",
	"413 5.867167862660949e-11\n",
	"414 5.855109452834739e-11\n",
	"415 5.7634869099487673e-11\n",
	"416 5.798097418852066e-11\n",
	"417 5.714314438298729e-11\n",
	"418 5.7525491314880384e-11\n",
	"419 5.753773499317383e-11\n",
	"420 5.72689846933816e-11\n",
	"421 5.631775948367057e-11\n",
	"422 5.598362051717487e-11\n",
	"423 5.5867376697049664e-11\n",
	"424 5.629419153052595e-11\n",
	"425 5.5520841396594633e-11\n",
	"426 5.46322084793438e-11\n",
	"427 5.4010414196614676e-11\n",
	"428 5.341607364761636e-11\n",
	"429 5.3519549902958374e-11\n",
	"430 5.247411186126705e-11\n",
	"431 5.307565845158457e-11\n",
	"432 5.2371048470112314e-11\n",
	"433 5.2449510012930745e-11\n",
	"434 5.1879195384074706e-11\n",
	"435 5.1574945703070085e-11\n",
	"436 5.1512395043973314e-11\n",
	"437 5.08322897663227e-11\n",
	"438 5.057060326052465e-11\n",
	"439 4.9787240302689995e-11\n",
	"440 4.823476340565236e-11\n",
	"441 4.8459992962879284e-11\n",
	"442 4.797246627719076e-11\n",
	"443 4.824013410953398e-11\n",
	"444 4.7559997606860804e-11\n",
	"445 4.690832444698145e-11\n",
	"446 4.619358368040949e-11\n",
	"447 4.583974866356755e-11\n",
	"448 4.605983650041168e-11\n",
	"449 4.5755635391664384e-11\n",
	"450 4.547212259509159e-11\n",
	"451 4.5017298916372184e-11\n",
	"452 4.5089352390670356e-11\n",
	"453 4.437321690642371e-11\n",
	"454 4.417576374149412e-11\n",
	"455 4.4186581477090314e-11\n",
	"456 4.324371069563959e-11\n",
	"457 4.303992579002269e-11\n",
	"458 4.266882333570088e-11\n",
	"459 4.2662408328286716e-11\n",
	"460 4.2061472360632735e-11\n",
	"461 4.153392560435343e-11\n",
	"462 4.1709347781138106e-11\n",
	"463 4.15860852698291e-11\n",
	"464 4.1531614952683427e-11\n",
	"465 4.106182407981329e-11\n",
	"466 4.0790346794716825e-11\n",
	"467 4.062897934753451e-11\n",
	"468 3.990976646384148e-11\n",
	"469 4.011613263799063e-11\n",
	"470 3.9666533946380866e-11\n",
	"471 3.945664281412853e-11\n",
	"472 3.835248785222234e-11\n",
	"473 3.8230120458226935e-11\n",
	"474 3.747497104300557e-11\n",
	"475 3.7175280215295814e-11\n",
	"476 3.713800100779707e-11\n",
	"477 3.680154792018442e-11\n",
	"478 3.743382687160235e-11\n",
	"479 3.680740781608627e-11\n",
	"480 3.6479482629081517e-11\n",
	"481 3.598047901287593e-11\n",
	"482 3.572271645158054e-11\n",
	"483 3.5526505348659754e-11\n",
	"484 3.541624979397362e-11\n",
	"485 3.4761606787503396e-11\n",
	"486 3.4769877949036854e-11\n",
	"487 3.460022199308632e-11\n",
	"488 3.453215144388899e-11\n",
	"489 3.4166957457726355e-11\n",
	"490 3.428968220475781e-11\n",
	"491 3.439980245101282e-11\n",
	"492 3.4364174700263206e-11\n",
	"493 3.4181008717881767e-11\n",
	"494 3.368537393466653e-11\n",
	"495 3.368653272994848e-11\n",
	"496 3.291827227469568e-11\n",
	"497 3.2805473615393765e-11\n",
	"498 3.3228246543171025e-11\n",
	"499 3.3127479925898484e-11\n"
	]
	}
	],
	"source": [
	"for t in range(500):\n",
	" # Forward pass: Compute predicted y by passing x to the model\n",
	" y_pred = model(x)\n",
	"\n",
	" # Compute and print loss\n",
	" loss = criterion(y_pred, y)\n",
	" print(t, loss.item())\n",
	"\n",
	" # Zero gradients, perform a backward pass, and update the weights.\n",
	" optimizer.zero_grad()\n",
	" loss.backward()\n",
	" optimizer.step()"
	]
	},
	{
	"cell_type": "code",
	"execution_count": null,
	"metadata": {},
	"outputs": [],
	"source": []
	}
	],
	"metadata": {
	"kernelspec": {
	"display_name": "Python 3",
	"language": "python",
	"name": "python3"
	},
	"language_info": {
	"codemirror_mode": {
	"name": "ipython",
	"version": 3
	},
	"file_extension": ".py",
	"mimetype": "text/x-python",
	"name": "python",
	"nbconvert_exporter": "python",
	"pygments_lexer": "ipython3",
	"version": "3.6.2"
	}
	},
	"nbformat": 4,
	"nbformat_minor": 2
	}