Skip to content

Instantly share code, notes, and snippets.

@JIElite

JIElite/clean_data_eli.ipynb

Created May 11, 2022 07:45
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save JIElite/78b966bf95e71c3cd0ea0e073a1a8c63 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save JIElite/78b966bf95e71c3cd0ea0e073a1a8c63 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
clean_data_eli.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "4484e9ff",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"pd.set_option('display.max_columns', None)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "41d55a2a",
"metadata": {},
"outputs": [],
"source": [
"# TODO 是不是因為清完資料，都市住宅分區的 unique 才會變少？\n",
"def clean_invalid(dataframe):\n",
" # Clean invalid data\n",
" dataframe = dataframe[(dataframe['交易標的']!='土地') & (~dataframe['交易標的'].isna())]\n",
" dataframe['Month'] = dataframe['交易年月日'].str[:-2].astype('float')\n",
" dataframe = dataframe.query(\"Month>=10601 and Month<=11003 \")\n",
" dataframe = dataframe[(dataframe.Month!=10600) & (dataframe.Month!=10700) & \\\n",
" (dataframe.Month!=10800) & (dataframe.Month!=10900) & (dataframe.Month!=11000)]\n",
" return dataframe"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6c42025b",
"metadata": {},
"outputs": [],
"source": [
"def encode_city_land_usage(x):\n",
" # Encode {'其他': 0, '住': 1, '農': 2, '工': 3, '商': 4, \n",
" # '住商': 5, '其他住宅': 6, nan: nan}\n",
" if isinstance(x, str):\n",
" if '其他' in x:\n",
" end_idx = x.find('(') if '(' in x else len(x) + 1\n",
" if '住商' in x[:end_idx] or '住宅商業' in x[:end_idx]:\n",
" return 5\n",
" if '住宅' in x[:end_idx] or '住' in x[:end_idx]:\n",
" return 6\n",
" return 0\n",
" if '住' in x:\n",
" return 1\n",
" elif '農' in x:\n",
" return 2\n",
" elif '工' in x:\n",
" return 3\n",
" elif '商' in x:\n",
" return 4\n",
" else:\n",
" return x"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "dfb0417b",
"metadata": {},
"outputs": [],
"source": [
"def is_multiple_usages(x):\n",
" # If there one of the following delimiters in the\n",
" # field of data['主要用途'], then we call such builing is\n",
" # multiple usage.\n",
" return '、' in x or '；' in x or '．' in x or \\\n",
" '.' in x or '‧' in x or '﹒' in x or \\\n",
" '，' in x or '﹐' in x or \\\n",
" ' ' in x or '　' in x"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8509ee4d",
"metadata": {},
"outputs": [],
"source": [
"def encode_main_usage(x):\n",
" if not isinstance(x, str):\n",
" # nan remains\n",
" return x\n",
" \n",
" living = lambda x: True if '住' in x else False\n",
" farm_related = lambda x: True if '農舍' in x or '農業' in x or \\\n",
" '畜牧' in x or '豬舍' in x or \\\n",
" '雞舍' in x or '羊舍' in x or \\\n",
" '牛舍' in x or '禽舍' in x or \\\n",
" '堆肥舍' in x else False\n",
" manufaturing = lambda x: True if '工場' in x or '工廠' in x or '廠房' in x else False\n",
" business = lambda x: True if '事務所' in x or '辦公' in x or '商業' in x else False\n",
" selling = lambda x: True if '店鋪' in x or '商店' in x or \\\n",
" '店舖' in x or '零售' in x or \\\n",
" '店房' in x or '商場' in x or \\\n",
" '百貨' in x else False\n",
" medical = lambda x: True if '醫' in x or '病' in x or '診所' in x else False\n",
" parking = lambda x: True if '停車' in x or '車庫' in x else False\n",
" others = lambda x: True if '金融' in x or '健身' in x else False\n",
" \n",
" # Ignore the comment in the field\n",
" end_idx = x.find('（') if '（' in x else len(x) - 1\n",
" x = x[:end_idx]\n",
"\n",
" if living(x): \n",
" if selling(x):\n",
" return 5\n",
" elif business(x):\n",
" return 4\n",
" elif manufaturing(x):\n",
" return 3\n",
" elif farm_related(x):\n",
" return 2\n",
" elif medical(x):\n",
" return 6\n",
" elif others(x):\n",
" return 8\n",
" else:\n",
" return 1\n",
" else:\n",
" if manufaturing(x):\n",
" return 3\n",
" elif business(x):\n",
" return 4\n",
" elif selling(x):\n",
" return 5\n",
" elif farm_related(x):\n",
" return 2\n",
" elif medical(x):\n",
" return 6\n",
" elif parking(x):\n",
" return 7\n",
" else:\n",
" # Cases list\n",
" # mutiple usage:\n",
" # - https://gist.github.com/JIElite/73a870e25390a23a676da3f9a2c86436\n",
" # single usage:\n",
" # - https://gist.github.com/JIElite/329341d1573088e389cbc84f38dd15ac\n",
" return 8"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "50922f04",
"metadata": {},
"outputs": [],
"source": [
"def encode_have_elevator(x):\n",
" \"\"\"Encode the value in '電梯' field with one-hot encoding\n",
" \n",
" Return:\n",
" - nan, nan\n",
" - 1, '有'\n",
" - 0, '無'\n",
" \"\"\"\n",
" if not isinstance(x, str):\n",
" return x\n",
" map_ = {'有': 1, '無': 0}\n",
" return map_[x]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d79ce245",
"metadata": {},
"outputs": [],
"source": [
"def transform_area(area):\n",
" \"\"\"Convert the unit of the area from square meter to Ping(坪)\n",
" \"\"\"\n",
" # If the original area is nan, then it remains.\n",
" if not isinstance(area, str):\n",
" return area\n",
" \n",
" return round(float(area) / 3.3058)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "4f45371e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"original rows: 234724\n"
]
}
],
"source": [
"sale_future_data = pd.read_csv('./sale_future_data.csv', dtype='str', encoding='utf-8')\n",
"print('original rows:', len(sale_future_data))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "65641a57",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"original rows: 1840485\n"
]
}
],
"source": [
"sale_data = pd.read_csv('./sale_data.csv', dtype='str', encoding='utf-8')\n",
"print('original rows:', len(sale_data))"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "32cea183",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"鄉鎮市區\n",
"交易標的\n",
"土地位置建物門牌\n",
"土地移轉總面積平方公尺\n",
"都市土地使用分區\n",
"非都市土地使用分區\n",
"非都市土地使用編定\n",
"交易年月日\n",
"交易筆棟數\n",
"移轉層次\n",
"總樓層數\n",
"建物型態\n",
"主要用途\n",
"主要建材\n",
"建築完成年月\n",
"建物移轉總面積平方公尺\n",
"建物現況格局-房\n",
"建物現況格局-廳\n",
"建物現況格局-衛\n",
"建物現況格局-隔間\n",
"有無管理組織\n",
"總價元\n",
"單價元平方公尺\n",
"車位類別\n",
"車位移轉總面積(平方公尺)\n",
"車位總價元\n",
"備註\n",
"編號\n",
"主建物面積\n",
"附屬建物面積\n",
"陽台面積\n",
"電梯\n",
"移轉編號\n"
]
}
],
"source": [
"for col in sale_data.columns:\n",
" print(col)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "afcefbb2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Before cleaning invalid data, the unique values for column:\n",
"-都市土地使用分區:1739\n",
"-主要用途:3173\n",
"-主要建材:823\n",
"-車位移轉總面積(平方公尺):10910\n",
"-電梯:3\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/tmp/ipykernel_673293/794416805.py:5: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame.\n",
"Try using .loc[row_indexer,col_indexer] = value instead\n",
"\n",
"See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
" dataframe['Month'] = dataframe['交易年月日'].str[:-2].astype('float')\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"--------------------\n",
"After cleaning invalid data, the unique values for column:\n",
"-都市土地使用分區:1005\n",
"-主要用途:1915\n",
"-主要建材:381\n",
"-車位移轉總面積(平方公尺):10208\n",
"-電梯:3\n"
]
}
],
"source": [
"my_columns = ['都市土地使用分區', '主要用途', '主要建材', '車位移轉總面積(平方公尺)', '電梯']\n",
"print('Before cleaning invalid data, the unique values for column:')\n",
"for column in my_columns:\n",
" print(f'-{column}:{len(sale_data[column].unique())}')\n",
" \n",
"sale_data = clean_invalid(sale_data)\n",
"print('-'*20)\n",
"print('After cleaning invalid data, the unique values for column:')\n",
"for column in my_columns:\n",
" print(f'-{column}:{len(sale_data[column].unique())}')\n",
"# sale_data = clean_invalid(sale_data)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "46bc9d72",
"metadata": {},
"outputs": [],
"source": [
"# Encode city land usage\n",
"print('Unique values for 都市土地使用分區 before encoding:', sale_data['都市土地使用分區'].unique(), \\\n",
" len(sale_data['都市土地使用分區'].unique()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8eec99b1",
"metadata": {},
"outputs": [],
"source": [
"sale_data['都市土地使用分區'] = sale_data['都市土地使用分區'].apply(encode_city_land_usage)\n",
"print('Unique values after encoding:', sale_data['都市土地使用分區'].unique(), \\\n",
" len(sale_data['都市土地使用分區'].unique()))"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "05c3ea51",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Unique values for 主要用途 before encoding: ['住家用' '停車空間' '商業用' ... 'Ｈ２集合住宅、停車空間' '供公眾使用停車空間等' '管理室、雞舍'] 1915\n"
]
}
],
"source": [
"print('Unique values for 主要用途 before encoding:', sale_data['主要用途'].unique(), \\\n",
" len(sale_data['主要用途'].unique()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5c01840e",
"metadata": {},
"outputs": [],
"source": [
"sale_data['主要用途'] = sale_data['主要用途'].apply(encode_main_usage)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bad0b096",
"metadata": {},
"outputs": [],
"source": [
"sale_data['電梯'].unique()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eab67b5e",
"metadata": {},
"outputs": [],
"source": [
"sale_data['電梯'].apply(encode_have_elevator)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "c8d1704d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 鋼筋混凝土造\n",
"1 加強磚造\n",
"2 見其他登記事項\n",
"3 鋼骨鋼筋混凝土造\n",
"4 磚造\n",
"5 木造\n",
"7 鋼筋混凝土加強磚造\n",
"8 見其它登記事項\n",
"9 土造\n",
"10 鋼筋混凝土構造\n",
"11 見使用執照\n",
"12 鋼骨混凝土造\n",
"13 鋼筋混凝土加強空心磚造\n",
"14 造\n",
"15 預力混凝土造\n",
"16 壁式預鑄鋼筋混凝土造\n",
"17 土磚石混合造\n",
"18 鋼造\n",
"19 ＲＣ造\n",
"20 鐵造\n",
"21 混凝土造\n",
"22 石造\n",
"23 竹造\n",
"24 鋼筋加強造\n",
"25 筋加強造\n",
"26 土木造\n",
"27 鋼筋混凝土構造，鋼鐵構造有牆\n",
"28 石\n",
"29 補強紅磚造\n",
"30 鋼骨構造\n",
"31 鋼骨造\n",
"32 空心磚造\n",
"33 鋼骨造。\n",
"34 石造\n",
"35 筋加強紅磚混凝土\n",
"36 鋼筋加強紅磚\n",
"37 鋼筋加強紅磚\n",
"38 加強磚造、鋼筋混凝土\n",
"39 磚瓦造\n",
"40 鋼筋加強空心磚造\n",
"41 混凝土構造\n",
"42 鋼筋混凝土\n",
"43 筋加強空心磚\n",
"44 鋼筋加強\n",
"45 鐵筋加強造\n",
"46 鋼筋水泥磚\n",
"47 磚石造\n",
"48 加強磚造、鋼鐵造。\n",
"49 水泥混凝土造\n",
"50 鋼鐵造。\n",
"51 補強磚　　　　\n",
"52 鋼骨造有牆\n",
"53 鋼架浪板有壁造\n",
"54 磚木造、ＲＣ造\n",
"55 磚木、加強磚造\n",
"56 磚木水泥柱石綿瓦造\n",
"57 鋼鐵造(無牆)\n",
"58 鋼骨構造、鋼鐵構造有牆\n",
"59 混凝土造\n",
"60 鋼骨構造有牆\n",
"61 R.C造\n",
"62 鋼筋混凝土造、鋼架造\n",
"63 ＲＣ構架造\n",
"64 磚木造、（增建）預鑄水泥柱磚木造，ＲＣ造，磚造\n",
"65 鋼骨構造、鋼鐵構造（有牆）、鋼鐵構造（無牆）\n",
"66 鋼筋混凝土構造，鋼骨構造，鋼鐵構造無牆\n",
"67 木石磚造〈磚石造〉\n",
"68 鋼鐵構造（有牆）（新建）；鋼鐵構造（有牆）（增建）\n",
"69 木石磚造\n",
"70 筋空心磚造\n",
"71 造\n",
"72 Ｒ．Ｃ造，鋼骨造。\n",
"73 鋼骨造、鋼筋混凝土造\n",
"74 磚木造\n",
"75 磚木、鋼鐵﹝有壁﹞、ＲＣ\n",
"76 鋼鐵造無牆\n",
"77 鋼骨構造、鋼筋混凝土構造\n",
"78 鋼骨結構，鋼鐵構造有牆，鋼鐵構造無牆，鋼鐵造，塑鋼造\n",
"79 鐵筋水泥磚\n",
"80 鐵筋水泥磚造\n",
"81 筋加強磚造\n",
"82 鋼骨造、鋼骨鋼筋混凝土造\n",
"83 加強磚造、水泥地床、水泥平頂\n",
"84 鋼骨混凝土構造\n",
"85 鋼骨鋼筋混凝土構造\n",
"86 鋼骨造、鋼架造、鋼筋混凝土造\n",
"87 鋼架造、磚造\n",
"88 磚造二層鋼筋補強\n",
"89 鋼筋混凝土造、鋼骨鋼筋混凝土造\n",
"90 ＲＣ造１３樓基礎\n",
"91 鋼鐵有牆造\n",
"92 鋼架造〈有牆〉\n",
"93 磚木、ＲＣ、鋼鐵有壁造\n",
"94 ＲＣ磚造、ＲＣ造、鋼骨造、木造\n",
"95 鋼鐵造有牆、鋼鐵造無牆\n",
"96 鋼骨或鋼骨混合造\n",
"97 鋼筋水泥加強磚造\n",
"98 鋼筋加強磚造\n",
"99 石及紅磚造\n",
"100 加強磚造蓋瓦頂及水泥頂\n",
"101 加強磚造、鋼造\n",
"102 鋼鐵構造有牆\n",
"103 鋼筋混凝土結構造\n",
"104 鋼骨構造（有牆）\n",
"105 Ｒ．Ｃ造\n",
"106 ＲＣ加強磚造\n",
"107 鋼骨ＲＣ造\n",
"108 ＲＣ造、鋼骨造\n",
"109 預鑄水泥柱磚木造、鋼鐵造\n",
"110 磚木造瓦屋頂\n",
"111 水泥磚造，鋼筋水泥\n",
"112 筋混凝造\n",
"113 水泥凝土造\n",
"114 ＲＣ鋼筋混凝土造\n",
"115 加強磚造、水泥平頂、磨石地床\n",
"116 加強磚造、磚石造\n",
"117 鐵筋加強磚造\n",
"118 加強磚造、水泥平頂、水泥地床\n",
"119 鐵筋磚造蓋瓦頂\n",
"120 鋼構造\n",
"121 鐵筋加強磚造瓦頂及水泥平頂\n",
"122 鋼骨鋼筋混凝土造，鋼筋混凝土造，鋼骨造\n",
"123 土竹造〈純土造〉、加強磚造\n",
"124 鋼筋混凝土造、鋼骨造\n",
"125 鋼鐵造\n",
"126 鋼骨構造，鋼筋混凝土構造\n",
"127 鐵骨造\n",
"128 鋼筋混凝土構造、鋼骨構造有牆\n",
"129 磚造、竹造\n",
"130 鋼筋混凝土、鋼骨構造\n",
"131 鋼骨鋼筋混凝土\n",
"132 鋼鐵造，鋼筋混凝土造\n",
"133 鋼骨結構造\n",
"134 鋼筋混凝土、鋼骨鋼筋混凝土構造\n",
"135 Ｒ．Ｃ構架\n",
"136 鋼骨構造、鋼骨鋼筋混凝土構造\n",
"137 磚木造．鋼架造．ＲＣ加強磚造\n",
"138 鋼骨造、鋼筋混凝土（ＲＣ）\n",
"139 鋼筋混凝土（ＲＣ）\n",
"140 鋼鐵造有牆\n",
"141 輕鋼架造\n",
"142 鋼架造、鋼筋混凝土造\n",
"143 鋼筋混凝土﹐鋼骨\n",
"144 磚造一層鋼筋補強\n",
"145 鋼骨\n",
"146 鐵筋混凝土磚造\n",
"147 加強磚造，鋼鐵無牆造\n",
"148 鋼骨水泥造加強磚造\n",
"149 鋼鐵木造磚造\n",
"150 雜木磚造\n",
"151 ＲＣ造、磚造\n",
"152 磚造ＲＣ補強\n",
"153 鋼鐵造有牆、鋼筋混凝土造\n",
"154 水泥磚造\n",
"155 磚木造、鐵筋加強磚造\n",
"156 加強磚造、鋼筋混凝土造\n",
"157 鐵架有避造\n",
"158 鋼筋混凝土加強磚木造\n",
"159 木構造、鋼鐵構造無牆、鋼骨構造、鋼筋混凝土造\n",
"160 磚造瓦頂\n",
"161 鋼架造\n",
"162 ＲＣ構造\n",
"163 加強磚造，鋼鐵構造無牆\n",
"164 鐵架蓋鋁皮\n",
"165 混凝土造\n",
"166 ︰煉瓦造\n",
"167 鋼骨鋼筋混凝土造；鋼骨造\n",
"168 鐵筋磚造\n",
"169 磚造、鋼筋混凝土\n",
"170 加強石造\n",
"171 煉瓦木混瓦葺造\n",
"172 加強磚造、磨石地床、水泥平頂\n",
"173 水泥瓦頂、水泥地床、磚造\n",
"174 ＲＣ鋼筋混凝土造、鋼骨造\n",
"175 加強磚造水泥頂\n",
"176 Ｒ‧Ｃ鋼筋混凝土造\n",
"177 磚造蓋瓦頂\n",
"178 木石磚造﹝磚石造﹞\n",
"179 加強磚造，鋼架\n",
"180 加強磚造、水泥平頂、磨石地床、磁磚地床\n",
"181 磨石地床、水泥平頂、加強磚造\n",
"182 水泥平頂、磨石地床、加強磚造\n",
"183 鋼骨造　\n",
"184 鋼架石棉瓦\n",
"185 鐵筋磚造蓋瓦頂及鉛板頂\n",
"186 鋼筋混凝土構造、鋼骨構造\n",
"187 鋼骨、ＲＣ造\n",
"188 ＳＲＣ、鋼骨造\n",
"189 加磚鋼架\n",
"190 ＲＣ造、加強磚造、鋼架造\n",
"191 加強磚造　鋼鐵造\n",
"192 木石磚造（磚石造）\n",
"193 筋加強磚造\n",
"194 ＲＣ造、鋼架造\n",
"195 鋼骨造、鋼架造\n",
"196 ＲＣ造，筋加強磚造\n",
"197 ＲＣ造，鋼骨造\n",
"198 鋼骨構造無牆\n",
"199 鐵骨、石綿瓦、加強磚造\n",
"200 加強磚木造\n",
"201 磚木\n",
"202 加強磚造、Ｒ．Ｃ造\n",
"203 一、二層加強磚造，三層ＲＣ造\n",
"204 鋼筋混凝土構造，鋼骨構造。\n",
"205 補強磚造\n",
"206 鋼筋水泥、鋼骨鋼筋混凝土造\n",
"207 鐵骨石棉瓦磚\n",
"208 鋼骨構造有牆。\n",
"209 鋼有壁造\n",
"210 磚造、石綿瓦\n",
"211 鐵骨造，加強磚造。\n",
"212 加強磚造鋼鐵造\n",
"213 鋼骨構造。\n",
"214 加強磚構造\n",
"215 ＲＣ造、ＳＲＣ\n",
"216 磚造、補強柱\n",
"217 鋼骨構造、鋼骨鋼筋混凝土構造、鋼筋混凝土造\n",
"218 ＲＣ結構造\n",
"219 鋼磚造、鋼筋混凝土造\n",
"220 鋼骨鋼筋混凝土構造，鋼筋混凝土\n",
"221 鋼骨構造、鋼骨鋼筋混凝土構造、鋼筋混凝土構造\n",
"222 鋼鐵造有牆者\n",
"223 鋼骨鋼筋混凝土構造、鋼筋混凝土\n",
"224 鋼骨構造〈有牆〉\n",
"225 鋼骨構造，鋼骨鋼筋混凝土構造，鋼筋混凝土造\n",
"226 Ｒ．Ｃ構架造\n",
"227 Ｒ．Ｃ構造\n",
"228 ＲＣ鋼架造\n",
"229 鋼架造、加強磚造\n",
"230 預鑄鋼筋混凝土造\n",
"231 ＲＣ構架\n",
"232 鋼骨鋼筋混凝土構造、鋼骨構造、鋼筋混凝土構造\n",
"233 鋼構、鋼筋混凝土造\n",
"234 加強鋼筋混凝土造\n",
"235 Ｒ．Ｃ結構\n",
"236 鋼骨造、鋼筋混凝土造（ＲＣ）\n",
"237 鋼骨造（ＳＣ）\n",
"238 鋼筋混凝土造（ＲＣ）\n",
"239 鋼筋混凝土（ＲＣ）、鋼鐵（骨）有牆造\n",
"240 鋼筋混凝土（ＲＣ）、鋼筋混凝土加強磚造\n",
"241 加強磚造、木石磚造（磚石造）\n",
"242 鋼鐵（骨）造有牆、鋼骨造\n",
"243 Ｒ．Ｃ造．無牆鋼架造\n",
"244 鋼筋混凝土造．鋼骨造\n",
"245 鋼筋混凝土造、加強磚造\n",
"246 鋼筋混凝土及加強磚造\n",
"247 鋼筋混泥土造、鋼鐵造\n",
"248 鋼架造，鋼筋混凝土\n",
"249 加強磚造、磚造\n",
"250 鋼架造、鋼筋混凝土\n",
"251 鋼骨鋼筋混凝土造、鋼筋混凝土造\n",
"252 磚造．加強磚造\n",
"253 加強磚造，木石磚造（磚石造）\n",
"254 鋼骨造、磚造\n",
"255 土塊造\n",
"256 磚石造、加強磚造\n",
"257 加強磚造、鋼架造、磚造\n",
"258 鋼架造、鋼筋混泥土\n",
"259 鋼筋混凝土造鋼骨\n",
"260 鋼管桁架、方管桁架、加強磚造、拱型６分錏管鍍鋅方管\n",
"261 鋼筋混凝土造、磚造\n",
"262 鋼筋混凝土，有牆鋼架造\n",
"263 鋼筋混凝土造、鋼鐵造\n",
"264 鋼筋補強磚木造\n",
"265 木石磚造（雜木）\n",
"266 鋼筋混凝土木磚造\n",
"267 鋼筋混凝土磚造\n",
"268 加強磚造、木石磚造\n",
"269 鋼鐵\n",
"270 鋼骨水泥加強磚造\n",
"271 加強磚造及鋼架，石棉瓦造\n",
"272 鐵筋混凝土磚造ＲＣ造\n",
"273 鐵筋混凝土及磚造\n",
"274 ＲＣ造，鋼架構造有牆\n",
"275 ＲＣ補強\n",
"276 鋼鐵造有（無）牆\n",
"277 鋼鐵造有牆、ＲＣ造\n",
"278 土木磚造\n",
"279 鋼鐵架（有牆者）\n",
"280 土磚造\n",
"281 ＲＣ鋼構造\n",
"282 鋼鐵造有牆、ＲＣ加強磚造\n",
"283 鋼架造﹝有牆﹞\n",
"284 鋼骨混凝土造，加強磚造\n",
"285 水泥柱造、有牆鋼鐵造、鋼筋混凝土造\n",
"286 鋼骨造，鋼骨鋼筋混凝土造，鋼筋混凝土造\n",
"287 鐵骨磚造\n",
"288 鋼筋混凝土結構造，鋼架造有牆\n",
"289 鋼構牆\n",
"290 鋼筋混凝土造、鋼板瓦\n",
"291 鐵筋混泥土加強磚造\n",
"292 加強磚造，有牆鋼架造\n",
"293 水泥柱造\n",
"294 鐵架磚造\n",
"295 有牆鋼鐵造\n",
"296 木石磚（磚石）造\n",
"297 磚木構造\n",
"298 鋼骨造、鋼骨鋼筋混凝土造、鋼筋混凝土造\n",
"299 鋼架造、有牆\n",
"300 加強磚造、鋼架有牆\n",
"301 加強磚造、ＲＣ造\n",
"302 加強磚造及ＲＣ造\n",
"303 加強磚造磚造鐵骨架\n",
"304 ＳＲＣ造\n",
"305 水泥板造\n",
"306 ＲＣ造、水泥柱磚木造、磚造\n",
"307 ＲＣ造、鋼骨造、鋼鐵造\n",
"308 ＲＣ鋼骨造\n",
"309 鋼筋混凝土造、輕鋼架造\n",
"310 泥磚造\n",
"311 鋼筋混凝土構造、鋼鐵構造無牆\n",
"312 三、四層鋼筋混泥土　一、二層加強磚造\n",
"313 磚造蓋石棉瓦．加強磚造\n",
"314 鋼鐵造﹝有墻﹞、鐵架造﹝有墻﹞\n",
"315 鋼骨構造、鋼鐵構造有牆、鋼鐵構造無牆\n",
"316 磚造平房、屋頂平頂\n",
"317 土木磚瓦\n",
"318 鋼筋混凝土鋼骨造、鋼骨構造、鋼筋混凝土構造\n",
"319 竹及土角造\n",
"320 鋼筋混凝土構架造\n",
"321 鋼骨構架造\n",
"322 磚造二樓，屋頂平頂\n",
"323 水泥鋼筋磚造\n",
"324 磚造蓋瓦\n",
"325 鋼鐵構造有牆、鋼鐵構造無牆\n",
"326 磚造屋頂平頂\n",
"327 鋼筋混凝土、木造\n",
"328 磚木造及木造\n",
"329 鋼筋水泥磚及木造\n",
"330 加強磚、木造\n",
"331 木造蓋鐵\n",
"332 鋼筋水泥磚造、ＲＣ加強磚造\n",
"333 鋼筋水泥磚蓋瓦\n",
"334 鋼筋水泥加強磚造、鋼架有牆\n",
"335 鋼筋水泥．加強磚造\n",
"336 水泥磚蓋瓦\n",
"337 鋼筋水泥\n",
"338 蓋瓦水泥磚木\n",
"339 水泥磚木蓋瓦\n",
"340 水泥磚木造蓋瓦\n",
"341 有牆鋼架造\n",
"342 ＲＣ\n",
"343 有墻鋼架造\n",
"344 有墻鋼鐵造\n",
"345 鋼筋水泥造\n",
"346 ＲＣ造、鋼架有墻造\n",
"347 筋加強磚\n",
"348 加強磚造、鋼架造\n",
"349 磚石造。\n",
"350 鋼架、加強磚造、鋼骨構造\n",
"351 鋼骨造、Ｒ．Ｃ造\n",
"352 鋼鐵造、磚木造\n",
"353 鐵造、石棉瓦\n",
"354 鋼骨構造、鋼筋混凝土造\n",
"355 鋼鐵構造（有牆）\n",
"356 鋼筋混凝土造、鋼筋混凝土造加強磚造\n",
"357 鋼鐵造有牆，鋼筋混凝土（ＲＣ）\n",
"358 鋼筋混凝土（ＲＣ）、鋼鐵（骨）造有牆\n",
"359 木磚造\n",
"360 ＲＣ造、ＲＣ加強磚造\n",
"361 鋼鐵造有牆、鋼骨造有牆\n",
"362 鋼架有牆、鋼骨造\n",
"363 加強磚造、鐵架造\n",
"364 鋼架造及磚造\n",
"365 ＲＣ磚木造\n",
"366 鋼骨造﹒ＲＣ造\n",
"367 鋼骨加強磚造\n",
"368 鋼骨構造、鋼鐵構造無牆\n",
"369 有牆鋼造架\n",
"370 ＲＣ造、鋼骨造（有牆）、鋼筋混凝土造\n",
"371 蓋瓦、土磚造\n",
"372 鋼鐵構造\n",
"373 鋼架水泥板造\n",
"374 加強磚造、鋼造\n",
"375 鋼架加強磚造\n",
"376 鋼筋混凝土造．鋼架造有牆\n",
"377 剛進混凝土構造\n",
"378 鋼鐵造、加強磚造\n",
"379 磚造、鐵筋\n",
"380 住宅\n"
]
}
],
"source": [
"for i, material in enumerate(sale_data['主要建材'].unique()):\n",
" if not isinstance(material, str):\n",
" continue\n",
" print(i, material)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "460268ba",
"metadata": {},
"outputs": [],
"source": [
"def replace(s):\n",
" if not isinstance(s, str):\n",
" return s\n",
" \n",
" if '\\u3000' in s:\n",
" s.replace('\\u3000', '')\n",
" if '\\ue04a' in s:\n",
" s.replace('\\ue04a', '鋼')\n",
"\n",
"material_map = {}\n",
"for material in sale_data['主要建材']:\n",
" if material not in material_map:\n",
" material_map[material] = 1\n",
" else:\n",
" material_map[material] += 1"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aee69016",
"metadata": {},
"outputs": [],
"source": [
"from operator import itemgetter\n",
"import matplotlib.pyplot as plt\n",
"import math\n",
"\n",
"material_map_items = sorted(material_map.items(), key=itemgetter(1), reverse=True)\n",
"material_statistics = []\n",
"material_name = []\n",
"for key, value in material_map_items:\n",
" print(key, value)\n",
" material_name.append(key)\n",
" material_statistics.append(int(value))\n",
"\n",
"material_name = np.array(material_name)\n",
"material_count = np.array(material_statistics)\n",
"material_count = np.log(material_count)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7c35fdef",
"metadata": {},
"outputs": [],
"source": [
"material_count.max()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf2e9316",
"metadata": {},
"outputs": [],
"source": [
"plt.bar(material_name, material_count)\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7bb8fc8f",
"metadata": {},
"outputs": [],
"source": [
"sale_data['車位移轉總面積(平方公尺)'].unique()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ac08936a",
"metadata": {},
"outputs": [],
"source": [
"sale_data['車位移轉總面積(平方公尺)'].apply(transform_area)"
]
},
{
"cell_type": "markdown",
"id": "d50a313b",
"metadata": {},
"source": [
"# Testing and Analysis of raw data"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7b551fb6",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"# Survey the main usage of the building\n",
"\n",
"values_of_man_usage = sale_data['主要用途'].sort_values()\n",
"for x in values_of_man_usage.unique():\n",
" \n",
" if not isinstance(x, str):\n",
" # nan remains\n",
" continue\n",
" \n",
" living = lambda x: True if '住' in x else False\n",
" farm_related = lambda x: True if '農舍' in x or '農業' in x or \\\n",
" '畜牧' in x or '豬舍' in x or \\\n",
" '雞舍' in x or '羊舍' in x or \\\n",
" '牛舍' in x or '禽舍' in x or \\\n",
" '堆肥舍' in x else False\n",
" manufaturing = lambda x: True if '工場' in x or '工廠' in x or '廠房' in x else False\n",
" business = lambda x: True if '事務所' in x or '辦公' in x or '商業' in x else False\n",
" selling = lambda x: True if '店鋪' in x or '商店' in x or \\\n",
" '店舖' in x or '零售' in x or \\\n",
" '店房' in x or '商場' in x or \\\n",
" '百貨' in x else False\n",
" medical = lambda x: True if '醫' in x or '病' in x or '診所' in x else False\n",
" parking = lambda x: True if '停車' in x or '車庫' in x else False\n",
" others = lambda x: True if '金融' in x or '健身' in x else False\n",
" \n",
" # 不考慮備註\n",
" end_idx = x.find('（') if '（' in x else len(x) - 1\n",
" x = x[:end_idx]\n",
" \n",
" if '、' in x or '；' in x or '．' in x or \\\n",
" '.' in x or '‧' in x or '﹒' in x or \\\n",
" '，' in x or '﹐' in x or \\\n",
" ' ' in x or '　' in x:\n",
" # Mutltiple usages\n",
" if living(x): \n",
" if selling(x):\n",
" pass\n",
" elif business(x):\n",
" pass\n",
" elif manufaturing(x):\n",
" pass\n",
" elif farm_related(x):\n",
" pass\n",
" elif medical(x):\n",
" pass\n",
" elif others(x):\n",
" pass\n",
" else:\n",
" pass\n",
" else:\n",
" if manufaturing(x):\n",
" pass\n",
" elif business(x):\n",
" pass\n",
" elif selling(x):\n",
" pass\n",
" elif farm_related(x):\n",
" pass\n",
" elif medical(x):\n",
" pass\n",
" elif parking(x):\n",
" pass\n",
" else:\n",
" pass\n",
" else:\n",
" if living(x):\n",
" if selling(x):\n",
" pass\n",
" elif business(x):\n",
" pass\n",
" elif manufaturing(x):\n",
" pass\n",
" elif farm_related(x):\n",
" pass\n",
" elif medical(x):\n",
" pass\n",
" elif others(x):\n",
" pass\n",
" else:\n",
" pass\n",
" else:\n",
" if manufaturing(x):\n",
" pass\n",
" elif business(x):\n",
" pass\n",
" elif selling(x):\n",
" pass\n",
" elif farm_related(x):\n",
" pass\n",
" elif medical(x):\n",
" pass\n",
" elif parking(x):\n",
" pass\n",
" else:\n",
" # Cases list:\n",
" print(x)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6620ea7c",
"metadata": {},
"outputs": [],
"source": [
"filtered_data = sale_data.loc[sale_data['主要用途'] == '見其他登記事項']\n",
"filtered_data.head()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment