Tensorpack

mem-consump.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Code to reproduce memory growing when looping over dataset with prefetch data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensorpack:  0.5.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "# Specify where located tensorpack datasets:\n",
    "os.environ['TENSORPACK_DATASET'] = \"/media/user/fast_storage/tensorpack_data/\"\n",
    "# Specify temporary folder for ZMQ pipe files\n",
    "os.environ['TENSORPACK_PIPEDIR'] = \"tmp_pipedir\"\n",
    "# !mkdir $TENSORPACK_PIPEDIR\n",
    "\n",
    "import numpy as np\n",
    "from tensorpack.utils.utils import fix_rng_seed\n",
    "from tensorpack.dataflow.dataset.cifar import Cifar10\n",
    "\n",
    "from tensorpack import __version__\n",
    "print(\"Tensorpack: \", __version__)\n",
    "\n",
    "seed = 12345\n",
    "fix_rng_seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorpack.dataflow import AugmentImageComponent, BatchData, PrefetchData, PrefetchDataZMQ\n",
    "from tensorpack.dataflow.imgaug import Resize, Flip, RandomOrderAug, RandomApplyAug, RandomChooseAug, Affine\n",
    "from tensorpack.dataflow.imgaug import Hue, Saturation, Brightness, Contrast, Gamma\n",
    "from tensorpack.dataflow.imgaug import MinMaxNormalize, ToFloat32, MeanVarianceNormalize\n",
    "from tensorpack.dataflow import SelectComponent, FixedSizeData\n",
    "from tensorpack.dataflow.imgaug import AugmentorList\n",
    "from tensorpack.dataflow.imgaug import Resize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "# Train augmenters:\n",
    "geom_augmenters = [\n",
    "    Resize(shape=(128, 128)),\n",
    "    RandomOrderAug([\n",
    "        Flip(horiz=True, prob=0.5),\n",
    "        Flip(vert=True, prob=0.5),        \n",
    "        RandomApplyAug(\n",
    "                Affine(scale=(0.95, 1.05), \n",
    "                       translate_frac=(0.05, 0.05), \n",
    "                       rotate_max_deg=45.0,\n",
    "                       shear=15.0,\n",
    "                       border=cv2.BORDER_REPLICATE, border_value=0.0),\n",
    "                prob=0.75\n",
    "        ),\n",
    "    ])\n",
    "]\n",
    "\n",
    "color_augmenters = [\n",
    "    RandomChooseAug([\n",
    "        Hue(range=(0.0, 180.0), rgb=True),\n",
    "        Saturation(alpha=0.5, rgb=True),\n",
    "        Brightness(delta=50),\n",
    "        Contrast(factor_range=(0.75, 1.25)),\n",
    "        Gamma()\n",
    "    ]),\n",
    "    ToFloat32(),    \n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 30074108 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m   498364 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2265816 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 31738032 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 30073976 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m   498536 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2265776 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 31737908 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 30072596 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m   499788 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2265904 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 31736528 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 30073260 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m   499464 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2265564 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 31737200 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!top | grep used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m[0925 06:38:06 @cifar.py:33]\u001b[0m Found cifar10 data in /media/user/fast_storage/tensorpack_data/cifar10_data.\n"
     ]
    }
   ],
   "source": [
    "train_ds = Cifar10('train')\n",
    "\n",
    "batch_size = 256\n",
    "train_batches = PrefetchData(train_ds, nr_prefetch=batch_size, nr_proc=1)\n",
    "aug_train_ds = AugmentImageComponent(train_ds, geom_augmenters, index=0, copy=False)\n",
    "aug_train_ds = AugmentImageComponent(aug_train_ds, color_augmenters, index=0, copy=False)  \n",
    "train_batches = BatchData(aug_train_ds, batch_size=batch_size, use_list=True, remainder=False)\n",
    "train_batches = PrefetchData(train_batches, nr_prefetch=batch_size, nr_proc=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 22999576 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m  7568192 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270520 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 24663556 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 15756736 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 14811096 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270456 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 17420760 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 15123344 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15444500 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270444 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16787368 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 15121424 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15446348 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270516 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16785448 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 15119908 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15447788 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270592 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16783940 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!top | grep used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_batches.reset_state()\n",
    "for batch_imgs, batch_labels in train_batches.get_data():    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 14573612 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15993844 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270832 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16237744 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 14573260 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15994172 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270856 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16237404 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 14572480 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15995128 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270680 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16236624 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Mem :\u001b[m\u001b[m\u001b[1m 32838288 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 14571692 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m 15995808 \u001b[m\u001b[mused,\u001b[m\u001b[m\u001b[1m  2270788 \u001b[m\u001b[mbuff/cache\u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n",
      "KiB Swap:\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mtotal,\u001b[m\u001b[m\u001b[1m 33447932 \u001b[m\u001b[mfree,\u001b[m\u001b[m\u001b[1m        0 \u001b[m\u001b[mused.\u001b[m\u001b[m\u001b[1m 16235844 \u001b[m\u001b[mavail Mem \u001b[m\u001b[m\u001b[m\u001b[m\u001b[K\n"
     ]
    }
   ],
   "source": [
    "!top | grep used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

tqdm.ipynb

tensorpack_affine_imgaug.ipynb