Skip to content

Instantly share code, notes, and snippets.

@gusgad

gusgad/preprocessing.ipynb

Last active November 16, 2017 12:05
Show Gist options
  • Save gusgad/5ce54ed82ba59d459dd485996fea31cb to your computer and use it in GitHub Desktop.
Save gusgad/5ce54ed82ba59d459dd485996fea31cb to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
preprocessing.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"import random"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Shape of sample_image: (1920, 1080)\n"
]
}
],
"source": [
"# creating a sample 1920x1080 image and filling it with sequential\n",
"# integers (instead of zeros) just for testing purposes\n",
"sample_image = np.linspace(0, 255, num=2073600, dtype='float16').reshape((1920, 1080))\n",
"print('Shape of sample_image:', sample_image.shape)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Shape of images: (100, 3, 1920, 1080)\n"
]
}
],
"source": [
"# creating a small 100 element image dataset, considering 3 color channels\n",
"# might throw a 'MemoryError' depending on your RAM, if so - decrease the number of elements\n",
"images = np.repeat(sample_image, [300])\n",
"images = images.reshape((100, 3, 1920, 1080))\n",
"print('Shape of images:', images.shape)\n",
"\n",
"# the code above basically creates white images turning into black as the numeber increases"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {},
"outputs": [],
"source": [
"crop_size = 512\n",
"\n",
"# image cropping function which slices random patches\n",
"def crop(image):\n",
" x = random.randrange(image.shape[1]-crop_size)\n",
" y = random.randrange(image.shape[2]-crop_size)\n",
" # printing the random coordiates for testing purposes\n",
" print('x:', x, 'y:', y)\n",
" \n",
" patch = image[:, x:x+crop_size, y:y+crop_size]\n",
" patch = patch.reshape(1, patch.shape[0], patch.shape[1], patch.shape[2])\n",
" return patch"
]
},
{
"cell_type": "code",
"execution_count": 50,
"metadata": {},
"outputs": [],
"source": [
"# yielding 32 element image batch at a time\n",
"def generator(images, batch_size=32):\n",
" # length will be the total number of elements\n",
" length = images.shape[0]\n",
" # we use index to check whether we reached the end\n",
" index = 0\n",
" while True:\n",
" images_batch = np.zeros((1, 3, 512, 512))\n",
" # if we did reach the end - start iterating from the beginning\n",
" if index + batch_size >= length:\n",
" index = 0\n",
" #if not - crop the given image, append to the batch while incrementing the index\n",
" for i in range(batch_size):\n",
" cropped_image = crop(images[index])\n",
" images_batch = np.concatenate((images_batch, cropped_image))\n",
" index = index + 1\n",
" yield images_batch[1:]"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {},
"outputs": [],
"source": [
"batch_size = 32\n",
"\n",
"image_generator = generator(images, batch_size)"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"x: 800 y: 99\n",
"x: 25 y: 490\n",
"x: 575 y: 171\n",
"x: 1125 y: 564\n",
"x: 980 y: 553\n",
"x: 1098 y: 112\n",
"x: 130 y: 84\n",
"x: 1210 y: 170\n",
"x: 1134 y: 541\n",
"x: 282 y: 381\n",
"x: 808 y: 357\n",
"x: 226 y: 153\n",
"x: 1217 y: 82\n",
"x: 610 y: 467\n",
"x: 1384 y: 311\n",
"x: 17 y: 567\n",
"x: 635 y: 313\n",
"x: 338 y: 94\n",
"x: 352 y: 367\n",
"x: 431 y: 162\n",
"x: 1405 y: 66\n",
"x: 441 y: 385\n",
"x: 1169 y: 116\n",
"x: 21 y: 165\n",
"x: 926 y: 557\n",
"x: 167 y: 450\n",
"x: 522 y: 77\n",
"x: 1102 y: 346\n",
"x: 698 y: 154\n",
"x: 272 y: 341\n",
"x: 1321 y: 55\n",
"x: 974 y: 429\n"
]
}
],
"source": [
"# calling this function will yield 32 elements each time\n",
"batch_output = next(image_generator)"
]
},
{
"cell_type": "code",
"execution_count": 53,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"(32, 3, 512, 512)\n"
]
}
],
"source": [
"# output shape must be 32x3x512x512\n",
"print(batch_output.shape)"
]
},
{
"cell_type": "code",
"execution_count": 54,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[[[ 0.35424805, 0.35424805, 0.35424805, ..., 0.35449219,\n",
" 0.35449219, 0.35449219],\n",
" [ 0.35449219, 0.35449219, 0.35449219, ..., 0.35473633,\n",
" 0.35473633, 0.35473633],\n",
" [ 0.35498047, 0.35498047, 0.35498047, ..., 0.35522461,\n",
" 0.35522461, 0.35522461],\n",
" ..., \n",
" [ 0.57958984, 0.57958984, 0.57958984, ..., 0.57958984,\n",
" 0.57958984, 0.57958984],\n",
" [ 0.58007812, 0.58007812, 0.58007812, ..., 0.58007812,\n",
" 0.58007812, 0.58007812],\n",
" [ 0.58007812, 0.58007812, 0.58007812, ..., 0.58056641,\n",
" 0.58056641, 0.58056641]],\n",
"\n",
" [[ 1.20410156, 1.20410156, 1.20410156, ..., 1.20410156,\n",
" 1.20410156, 1.20410156],\n",
" [ 1.20410156, 1.20410156, 1.20410156, ..., 1.20507812,\n",
" 1.20507812, 1.20507812],\n",
" [ 1.20507812, 1.20507812, 1.20507812, ..., 1.20507812,\n",
" 1.20507812, 1.20507812],\n",
" ..., \n",
" [ 1.4296875 , 1.4296875 , 1.4296875 , ..., 1.4296875 ,\n",
" 1.4296875 , 1.4296875 ],\n",
" [ 1.4296875 , 1.4296875 , 1.4296875 , ..., 1.43066406,\n",
" 1.43066406, 1.43066406],\n",
" [ 1.43066406, 1.43066406, 1.43066406, ..., 1.43066406,\n",
" 1.43066406, 1.43066406]],\n",
"\n",
" [[ 2.0546875 , 2.0546875 , 2.0546875 , ..., 2.0546875 ,\n",
" 2.0546875 , 2.0546875 ],\n",
" [ 2.0546875 , 2.0546875 , 2.0546875 , ..., 2.0546875 ,\n",
" 2.0546875 , 2.0546875 ],\n",
" [ 2.0546875 , 2.0546875 , 2.0546875 , ..., 2.0546875 ,\n",
" 2.0546875 , 2.0546875 ],\n",
" ..., \n",
" [ 2.27929688, 2.27929688, 2.27929688, ..., 2.27929688,\n",
" 2.27929688, 2.27929688],\n",
" [ 2.27929688, 2.27929688, 2.27929688, ..., 2.27929688,\n",
" 2.27929688, 2.27929688],\n",
" [ 2.28125 , 2.28125 , 2.28125 , ..., 2.28125 ,\n",
" 2.28125 , 2.28125 ]]],\n",
"\n",
"\n",
" [[[ 2.56054688, 2.56054688, 2.56054688, ..., 2.56054688,\n",
" 2.56054688, 2.56054688],\n",
" [ 2.5625 , 2.5625 , 2.5625 , ..., 2.5625 ,\n",
" 2.5625 , 2.5625 ],\n",
" [ 2.5625 , 2.5625 , 2.5625 , ..., 2.5625 ,\n",
" 2.5625 , 2.5625 ],\n",
" ..., \n",
" [ 2.78710938, 2.78710938, 2.78710938, ..., 2.78710938,\n",
" 2.78710938, 2.78710938],\n",
" [ 2.78710938, 2.78710938, 2.78710938, ..., 2.78710938,\n",
" 2.78710938, 2.78710938],\n",
" [ 2.78710938, 2.78710938, 2.78710938, ..., 2.78710938,\n",
" 2.78710938, 2.78710938]],\n",
"\n",
" [[ 3.41210938, 3.41210938, 3.41210938, ..., 3.41210938,\n",
" 3.41210938, 3.41210938],\n",
" [ 3.41210938, 3.41210938, 3.41210938, ..., 3.41210938,\n",
" 3.41210938, 3.41210938],\n",
" [ 3.41210938, 3.41210938, 3.41210938, ..., 3.41210938,\n",
" 3.41210938, 3.41210938],\n",
" ..., \n",
" [ 3.63671875, 3.63671875, 3.63671875, ..., 3.63671875,\n",
" 3.63671875, 3.63671875],\n",
" [ 3.63671875, 3.63671875, 3.63671875, ..., 3.63671875,\n",
" 3.63671875, 3.63671875],\n",
" [ 3.63671875, 3.63671875, 3.63671875, ..., 3.63671875,\n",
" 3.63671875, 3.63671875]],\n",
"\n",
" [[ 4.26171875, 4.26171875, 4.26171875, ..., 4.26171875,\n",
" 4.26171875, 4.26171875],\n",
" [ 4.26171875, 4.26171875, 4.26171875, ..., 4.26171875,\n",
" 4.26171875, 4.26171875],\n",
" [ 4.26171875, 4.26171875, 4.26171875, ..., 4.26171875,\n",
" 4.26171875, 4.26171875],\n",
" ..., \n",
" [ 4.48828125, 4.48828125, 4.48828125, ..., 4.48828125,\n",
" 4.48828125, 4.48828125],\n",
" [ 4.48828125, 4.48828125, 4.48828125, ..., 4.48828125,\n",
" 4.48828125, 4.48828125],\n",
" [ 4.48828125, 4.48828125, 4.48828125, ..., 4.48828125,\n",
" 4.48828125, 4.48828125]]],\n",
"\n",
"\n",
" [[[ 5.35546875, 5.35546875, 5.35546875, ..., 5.35546875,\n",
" 5.35546875, 5.35546875],\n",
" [ 5.35546875, 5.35546875, 5.35546875, ..., 5.35546875,\n",
" 5.35546875, 5.35546875],\n",
" [ 5.35546875, 5.35546875, 5.35546875, ..., 5.35546875,\n",
" 5.35546875, 5.35546875],\n",
" ..., \n",
" [ 5.578125 , 5.578125 , 5.578125 , ..., 5.58203125,\n",
" 5.58203125, 5.58203125],\n",
" [ 5.58203125, 5.58203125, 5.58203125, ..., 5.58203125,\n",
" 5.58203125, 5.58203125],\n",
" [ 5.58203125, 5.58203125, 5.58203125, ..., 5.58203125,\n",
" 5.58203125, 5.58203125]],\n",
"\n",
" [[ 6.203125 , 6.203125 , 6.203125 , ..., 6.203125 ,\n",
" 6.203125 , 6.203125 ],\n",
" [ 6.203125 , 6.203125 , 6.203125 , ..., 6.20703125,\n",
" 6.20703125, 6.20703125],\n",
" [ 6.20703125, 6.20703125, 6.20703125, ..., 6.20703125,\n",
" 6.20703125, 6.20703125],\n",
" ..., \n",
" [ 6.4296875 , 6.4296875 , 6.4296875 , ..., 6.4296875 ,\n",
" 6.4296875 , 6.4296875 ],\n",
" [ 6.4296875 , 6.4296875 , 6.4296875 , ..., 6.4296875 ,\n",
" 6.4296875 , 6.4296875 ],\n",
" [ 6.4296875 , 6.4296875 , 6.4296875 , ..., 6.4296875 ,\n",
" 6.4296875 , 6.4296875 ]],\n",
"\n",
" [[ 7.0546875 , 7.0546875 , 7.0546875 , ..., 7.0546875 ,\n",
" 7.0546875 , 7.0546875 ],\n",
" [ 7.0546875 , 7.0546875 , 7.0546875 , ..., 7.0546875 ,\n",
" 7.0546875 , 7.0546875 ],\n",
" [ 7.0546875 , 7.0546875 , 7.0546875 , ..., 7.0546875 ,\n",
" 7.0546875 , 7.0546875 ],\n",
" ..., \n",
" [ 7.28125 , 7.28125 , 7.28125 , ..., 7.28125 ,\n",
" 7.28125 , 7.28125 ],\n",
" [ 7.28125 , 7.28125 , 7.28125 , ..., 7.28125 ,\n",
" 7.28125 , 7.28125 ],\n",
" [ 7.28125 , 7.28125 , 7.28125 , ..., 7.28125 ,\n",
" 7.28125 , 7.28125 ]]],\n",
"\n",
"\n",
" ..., \n",
" [[[ 74.0625 , 74.0625 , 74.0625 , ..., 74.0625 ,\n",
" 74.0625 , 74.0625 ],\n",
" [ 74.0625 , 74.0625 , 74.0625 , ..., 74.0625 ,\n",
" 74.0625 , 74.0625 ],\n",
" [ 74.0625 , 74.0625 , 74.0625 , ..., 74.0625 ,\n",
" 74.0625 , 74.0625 ],\n",
" ..., \n",
" [ 74.3125 , 74.3125 , 74.3125 , ..., 74.3125 ,\n",
" 74.3125 , 74.3125 ],\n",
" [ 74.3125 , 74.3125 , 74.3125 , ..., 74.3125 ,\n",
" 74.3125 , 74.3125 ],\n",
" [ 74.3125 , 74.3125 , 74.3125 , ..., 74.3125 ,\n",
" 74.3125 , 74.3125 ]],\n",
"\n",
" [[ 74.9375 , 74.9375 , 74.9375 , ..., 74.9375 ,\n",
" 74.9375 , 74.9375 ],\n",
" [ 74.9375 , 74.9375 , 74.9375 , ..., 74.9375 ,\n",
" 74.9375 , 74.9375 ],\n",
" [ 74.9375 , 74.9375 , 74.9375 , ..., 74.9375 ,\n",
" 74.9375 , 74.9375 ],\n",
" ..., \n",
" [ 75.125 , 75.125 , 75.125 , ..., 75.125 ,\n",
" 75.125 , 75.125 ],\n",
" [ 75.125 , 75.125 , 75.125 , ..., 75.125 ,\n",
" 75.125 , 75.125 ],\n",
" [ 75.125 , 75.125 , 75.125 , ..., 75.125 ,\n",
" 75.125 , 75.125 ]],\n",
"\n",
" [[ 75.75 , 75.75 , 75.75 , ..., 75.75 ,\n",
" 75.75 , 75.75 ],\n",
" [ 75.75 , 75.75 , 75.75 , ..., 75.75 ,\n",
" 75.75 , 75.75 ],\n",
" [ 75.75 , 75.75 , 75.75 , ..., 75.75 ,\n",
" 75.75 , 75.75 ],\n",
" ..., \n",
" [ 76. , 76. , 76. , ..., 76. ,\n",
" 76. , 76. ],\n",
" [ 76. , 76. , 76. , ..., 76. ,\n",
" 76. , 76. ],\n",
" [ 76. , 76. , 76. , ..., 76. ,\n",
" 76. , 76. ]]],\n",
"\n",
"\n",
" [[[ 77.0625 , 77.0625 , 77.0625 , ..., 77.0625 ,\n",
" 77.0625 , 77.0625 ],\n",
" [ 77.0625 , 77.0625 , 77.0625 , ..., 77.0625 ,\n",
" 77.0625 , 77.0625 ],\n",
" [ 77.0625 , 77.0625 , 77.0625 , ..., 77.0625 ,\n",
" 77.0625 , 77.0625 ],\n",
" ..., \n",
" [ 77.3125 , 77.3125 , 77.3125 , ..., 77.3125 ,\n",
" 77.3125 , 77.3125 ],\n",
" [ 77.3125 , 77.3125 , 77.3125 , ..., 77.3125 ,\n",
" 77.3125 , 77.3125 ],\n",
" [ 77.3125 , 77.3125 , 77.3125 , ..., 77.3125 ,\n",
" 77.3125 , 77.3125 ]],\n",
"\n",
" [[ 77.9375 , 77.9375 , 77.9375 , ..., 77.9375 ,\n",
" 77.9375 , 77.9375 ],\n",
" [ 77.9375 , 77.9375 , 77.9375 , ..., 77.9375 ,\n",
" 77.9375 , 77.9375 ],\n",
" [ 77.9375 , 77.9375 , 77.9375 , ..., 77.9375 ,\n",
" 77.9375 , 77.9375 ],\n",
" ..., \n",
" [ 78.1875 , 78.1875 , 78.1875 , ..., 78.1875 ,\n",
" 78.1875 , 78.1875 ],\n",
" [ 78.1875 , 78.1875 , 78.1875 , ..., 78.1875 ,\n",
" 78.1875 , 78.1875 ],\n",
" [ 78.1875 , 78.1875 , 78.1875 , ..., 78.1875 ,\n",
" 78.1875 , 78.1875 ]],\n",
"\n",
" [[ 78.8125 , 78.8125 , 78.8125 , ..., 78.8125 ,\n",
" 78.8125 , 78.8125 ],\n",
" [ 78.8125 , 78.8125 , 78.8125 , ..., 78.8125 ,\n",
" 78.8125 , 78.8125 ],\n",
" [ 78.8125 , 78.8125 , 78.8125 , ..., 78.8125 ,\n",
" 78.8125 , 78.8125 ],\n",
" ..., \n",
" [ 79. , 79. , 79. , ..., 79. ,\n",
" 79. , 79. ],\n",
" [ 79. , 79. , 79. , ..., 79. ,\n",
" 79. , 79. ],\n",
" [ 79. , 79. , 79. , ..., 79. ,\n",
" 79. , 79. ]]],\n",
"\n",
"\n",
" [[[ 0. , 0. , 0. , ..., 79.5 ,\n",
" 79.5 , 79.5 ],\n",
" [ 0. , 0. , 0. , ..., 79.5 ,\n",
" 79.5 , 79.5 ],\n",
" [ 0. , 0. , 0. , ..., 79.5 ,\n",
" 79.5 , 79.5 ],\n",
" ..., \n",
" [ 79.6875 , 79.6875 , 79.6875 , ..., 79.6875 ,\n",
" 79.6875 , 79.6875 ],\n",
" [ 79.6875 , 79.6875 , 79.6875 , ..., 79.6875 ,\n",
" 79.6875 , 79.6875 ],\n",
" [ 79.6875 , 79.6875 , 79.6875 , ..., 79.6875 ,\n",
" 79.6875 , 79.6875 ]],\n",
"\n",
" [[ 0. , 0. , 0. , ..., 80.3125 ,\n",
" 80.3125 , 80.3125 ],\n",
" [ 0. , 0. , 0. , ..., 80.3125 ,\n",
" 80.3125 , 80.3125 ],\n",
" [ 0. , 0. , 0. , ..., 80.3125 ,\n",
" 80.3125 , 80.3125 ],\n",
" ..., \n",
" [ 80.5625 , 80.5625 , 80.5625 , ..., 80.5625 ,\n",
" 80.5625 , 80.5625 ],\n",
" [ 80.5625 , 80.5625 , 80.5625 , ..., 80.5625 ,\n",
" 80.5625 , 80.5625 ],\n",
" [ 80.5625 , 80.5625 , 80.5625 , ..., 80.5625 ,\n",
" 80.5625 , 80.5625 ]],\n",
"\n",
" [[ 0. , 0. , 0. , ..., 81.1875 ,\n",
" 81.1875 , 81.1875 ],\n",
" [ 0. , 0. , 0. , ..., 81.1875 ,\n",
" 81.1875 , 81.1875 ],\n",
" [ 0. , 0. , 0. , ..., 81.1875 ,\n",
" 81.1875 , 81.1875 ],\n",
" ..., \n",
" [ 81.4375 , 81.4375 , 81.4375 , ..., 81.4375 ,\n",
" 81.4375 , 81.4375 ],\n",
" [ 81.4375 , 81.4375 , 81.4375 , ..., 81.4375 ,\n",
" 81.4375 , 81.4375 ],\n",
" [ 81.4375 , 81.4375 , 81.4375 , ..., 81.4375 ,\n",
" 81.4375 , 81.4375 ]]]])"
]
},
"execution_count": 54,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"batch_output"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment