Skip to content

Instantly share code, notes, and snippets.

@yufengg

yufengg/AutoML_data_preparation_AIA023.ipynb

Last active October 9, 2025 07:18
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save yufengg/984ed8c02d95ce7e95e1c39da906ee04 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save yufengg/984ed8c02d95ce7e95e1c39da906ee04 to your computer and use it in GitHub Desktop.
Download ZIP
Notebook for preparing a CSV for Google Cloud AutoML Vision
Display the source blob
Display the rendered blob
Raw
AutoML_data_preparation_AIA023.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 112,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 104,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['chair_yellow',\n",
" 'chair_lime',\n",
" 'bike',\n",
" 'chair_red',\n",
" 'table',\n",
" 'chair_white',\n",
" 'chair_black',\n",
" 'chair_blue']"
]
},
"execution_count": 104,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# hardcoded\n",
"data_folders = ['chair_yellow',\n",
" 'chair_lime',\n",
" 'bike',\n",
" 'chair_red',\n",
" 'table',\n",
" 'chair_white',\n",
" 'chair_black',\n",
" 'chair_blue']\n",
"data_folders"
]
},
{
"cell_type": "code",
"execution_count": 113,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"chair_yellow150.jpg\n",
"chair_lime293.jpg\n",
"bike612.jpg\n",
"chair_red526.jpg\n",
"table026.jpg\n",
"chair_white076.jpg\n",
"chair_black157.jpg\n",
"chair_blue104.jpg\n"
]
},
{
"data": {
"text/plain": [
"[673, 729, 782, 751, 680, 742, 689, 672]"
]
},
"execution_count": 113,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# array of arrays, containing the list files, grouped by folder\n",
"filenames = [os.listdir(f) for f in data_folders]\n",
"[print(f[1]) for f in filenames]\n",
"[len(f) for f in filenames]"
]
},
{
"cell_type": "code",
"execution_count": 114,
"metadata": {},
"outputs": [],
"source": [
"files_dict = dict(zip(data_folders, filenames))"
]
},
{
"cell_type": "code",
"execution_count": 107,
"metadata": {},
"outputs": [],
"source": [
"base_gcs_path = 'gs://cloudml-demo-vcm/chairs_table_bike/'"
]
},
{
"cell_type": "code",
"execution_count": 108,
"metadata": {},
"outputs": [],
"source": [
"# What we want:\n",
"# gs://cloudml-demo-vcm/chairs_table_bike/chair_black/chair_black157.jpg, 'chair_black' \n",
"# base_gcs_path + dict_key + '/' + filename\n",
"\n",
"data_array = []\n",
"\n",
"for (dict_key, files_list) in files_dict.items():\n",
" for filename in files_list:\n",
"# print(base_gcs_path + dict_key + '/' + filename)\n",
" if '.jpg' not in filename: \n",
" continue # don't include non-photos\n",
"\n",
" label = dict_key\n",
"# label = 'chair' if 'chair' in dict_key else dict_key # for grouping all chairs as one label\n",
" \n",
" data_array.append((base_gcs_path + dict_key + '/' + filename , label))"
]
},
{
"cell_type": "code",
"execution_count": 115,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow636.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow150.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow144.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow622.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow178.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow193.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow187.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow352.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow434.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow420.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow346.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow408.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow391.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow385.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow226.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow540.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow554.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow232.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow568.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow583.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow597.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow024.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow030.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow018.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow019.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow031.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow025.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow596.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow582.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow569.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow555.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow233.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow227.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow541.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow384.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow390.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow409.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow421.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow347.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow353.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow435.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow186.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow192.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow179.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow145.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow623.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow637.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow151.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow621.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow147.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow153.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow635.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow609.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow184.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow190.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow345.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow423.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow437.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow351.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow379.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow386.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow392.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow231.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow557.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow543.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow225.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow219.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow594.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow580.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow033.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow027.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow026.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow032.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow581.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow595.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow218.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow542.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow224.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow230.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow556.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow393.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow387.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow378.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow436.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow350.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow344.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow422.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow191.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow185.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow608.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow152.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow634.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow620.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow146.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow618.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow142.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow624.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow630.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow156.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow181.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow195.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow368.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow426.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow340.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow354.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow432.jpg',\n",
" 'chair_yellow'),\n",
" ('gs://cloudml-demo-vcm/chairs_table_bike/chair_yellow/chair_yellow383.jpg',\n",
" 'chair_yellow'),\n",
" ...]"
]
},
"execution_count": 115,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data_array"
]
},
{
"cell_type": "code",
"execution_count": 116,
"metadata": {},
"outputs": [],
"source": [
"dataframe = pd.DataFrame(data_array)"
]
},
{
"cell_type": "code",
"execution_count": 117,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>0</th>\n",
" <th>1</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_yellow</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5679</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5680</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5681</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5682</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5683</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5684</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5685</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5686</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5687</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5688</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5689</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5690</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5691</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5692</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5693</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5694</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5695</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5696</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5697</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5698</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5699</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5700</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5701</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5702</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5703</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5704</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5705</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5706</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5707</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5708</th>\n",
" <td>gs://cloudml-demo-vcm/chairs_table_bike/chair_...</td>\n",
" <td>chair_blue</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5709 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" 0 1\n",
"0 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"1 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"2 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"3 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"4 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"5 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"6 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"7 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"8 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"9 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"10 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"11 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"12 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"13 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"14 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"15 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"16 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"17 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"18 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"19 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"20 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"21 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"22 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"23 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"24 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"25 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"26 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"27 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"28 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"29 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_yellow\n",
"... ... ...\n",
"5679 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5680 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5681 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5682 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5683 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5684 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5685 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5686 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5687 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5688 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5689 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5690 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5691 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5692 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5693 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5694 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5695 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5696 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5697 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5698 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5699 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5700 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5701 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5702 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5703 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5704 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5705 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5706 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5707 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"5708 gs://cloudml-demo-vcm/chairs_table_bike/chair_... chair_blue\n",
"\n",
"[5709 rows x 2 columns]"
]
},
"execution_count": 117,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dataframe"
]
},
{
"cell_type": "code",
"execution_count": 111,
"metadata": {},
"outputs": [],
"source": [
"dataframe.to_csv('all_data.csv', index=False, header=False)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"gsutil cp all_data.csv gs://cloudml-demo-vcm/chairs_table_bike/"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
@StraigenDaigen
Copy link

StraigenDaigen commented Feb 10, 2019

Can you help what I need to change to use this code from datalab?, I uploaded the folder (data) to Cloud Storage but I received some errors

@chelseawmk
Copy link

chelseawmk commented May 12, 2020

I think the csv input needs an additional column that defines the TRAIN, TEST, or VALIDATION group of the images.

@okwesi
Copy link

okwesi commented Aug 4, 2021

Can you help..... I want to change the code and use it for my local data instead of google cloud

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment