Skip to content

Instantly share code, notes, and snippets.

@taldcroft

taldcroft/join_masked.ipynb

Created July 30, 2014 20:57
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save taldcroft/65ecffa34aea59065873 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save taldcroft/65ecffa34aea59065873 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
join_masked.ipynb
{
"metadata": {
"name": "",
"signature": "sha256:d96e5c7eef189bf331586d0ec65e9ae4ec62217ee4a0c60d93e60b1d7728f12a"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "code",
"collapsed": false,
"input": [
"adat = \"\"\"astr1 | KeyName | KeyName2\n",
"A02 | N121 | abc\n",
"A219 | N115 | def \n",
"A219 | N115 | ghi\"\"\""
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"bdat = \"\"\"KeyName bStr KeyName2\n",
"N111 CO4 ghi\n",
"N111 CO5 jkl\n",
"N115 WN4 mno\"\"\""
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cdat = \"\"\"KeyName cfloat1 cfloat2 \n",
"N131 8.2 2.1\n",
"- 1.8 - \n",
"N115 8.3 2.5\"\"\""
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 3
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from astropy.io import ascii\n",
"from astropy.table import Table, join"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"f1 = ascii.read(adat)\n",
"f2 = ascii.read(bdat)\n",
"f3 = ascii.read(cdat, fill_values=[('-', '0')])"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 5
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"f1"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<table id=\"table4392517712\"><thead><tr><th>astr1</th><th>KeyName</th><th>KeyName2</th></tr></thead><tr><td>A02</td><td>N121</td><td>abc</td></tr><tr><td>A219</td><td>N115</td><td>def</td></tr><tr><td>A219</td><td>N115</td><td>ghi</td></tr></table>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 6,
"text": [
"<Table rows=3 names=('astr1','KeyName','KeyName2')>\n",
"array([('A02', 'N121', 'abc'), ('A219', 'N115', 'def'),\n",
" ('A219', 'N115', 'ghi')], \n",
" dtype=[('astr1', 'S4'), ('KeyName', 'S4'), ('KeyName2', 'S3')])"
]
}
],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"f2"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<table id=\"table4392519120\"><thead><tr><th>KeyName</th><th>bStr</th><th>KeyName2</th></tr></thead><tr><td>N111</td><td>CO4</td><td>ghi</td></tr><tr><td>N111</td><td>CO5</td><td>jkl</td></tr><tr><td>N115</td><td>WN4</td><td>mno</td></tr></table>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 7,
"text": [
"<Table rows=3 names=('KeyName','bStr','KeyName2')>\n",
"array([('N111', 'CO4', 'ghi'), ('N111', 'CO5', 'jkl'),\n",
" ('N115', 'WN4', 'mno')], \n",
" dtype=[('KeyName', 'S4'), ('bStr', 'S3'), ('KeyName2', 'S3')])"
]
}
],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"f3"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<table id=\"table4392518480\"><thead><tr><th>KeyName</th><th>cfloat1</th><th>cfloat2</th></tr></thead><tr><td>N131</td><td>8.2</td><td>2.1</td></tr><tr><td>--</td><td>1.8</td><td>--</td></tr><tr><td>N115</td><td>8.3</td><td>2.5</td></tr></table>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 8,
"text": [
"<Table rows=3 names=('KeyName','cfloat1','cfloat2')>\n",
"masked_array(data = [('N131', 8.2, 2.1) (--, 1.8, --) ('N115', 8.3, 2.5)],\n",
" mask = [(False, False, False) (True, False, True) (False, False, False)],\n",
" fill_value = ('N/A', 1e+20, 1e+20),\n",
" dtype = [('KeyName', 'S4'), ('cfloat1', '<f8'), ('cfloat2', '<f8')])"
]
}
],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"j = join(f1, f2, join_type='outer', keys='KeyName')\n",
"j2 = join(j, f3, join_type='outer', keys='KeyName')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "TableMergeError",
"evalue": "Right key column 'KeyName' has missing values",
"output_type": "pyerr",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTableMergeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-9-f596f177ae90>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'outer'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'KeyName'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mj2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mjoin_type\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'outer'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'KeyName'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/Users/aldcroft/git/astropy/astropy/table/operations.pyc\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(left, right, keys, join_type, uniq_col_name, table_names, metadata_conflicts)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[0mcol_name_map\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mOrderedDict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 148\u001b[0m out_data = np_utils.join(left._data, right._data, keys, join_type,\n\u001b[0;32m--> 149\u001b[0;31m uniq_col_name, table_names, col_name_map)\n\u001b[0m\u001b[1;32m 150\u001b[0m \u001b[0;31m# Create the output (Table or subclass of Table)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 151\u001b[0m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/Users/aldcroft/git/astropy/astropy/table/np_utils.pyc\u001b[0m in \u001b[0;36mjoin\u001b[0;34m(left, right, keys, join_type, uniq_col_name, table_names, col_name_map)\u001b[0m\n\u001b[1;32m 215\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'mask'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 216\u001b[0m raise TableMergeError('{0} key column {1!r} has missing values'\n\u001b[0;32m--> 217\u001b[0;31m .format(arr_label, name))\n\u001b[0m\u001b[1;32m 218\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 219\u001b[0m \u001b[0;31m# Make sure we work with ravelled arrays\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mTableMergeError\u001b[0m: Right key column 'KeyName' has missing values"
]
}
],
"prompt_number": 9
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cdat = \"\"\"KeyName cfloat1 cfloat2 \n",
"N131 8.2 2.1\n",
"N133 1.8 - \n",
"N115 8.3 2.5\"\"\"\n",
"f3 = ascii.read(cdat, fill_values=[('-', '0')])\n",
"f3"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<table id=\"table4393063312\"><thead><tr><th>KeyName</th><th>cfloat1</th><th>cfloat2</th></tr></thead><tr><td>N131</td><td>8.2</td><td>2.1</td></tr><tr><td>N133</td><td>1.8</td><td>--</td></tr><tr><td>N115</td><td>8.3</td><td>2.5</td></tr></table>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 10,
"text": [
"<Table rows=3 names=('KeyName','cfloat1','cfloat2')>\n",
"masked_array(data = [('N131', 8.2, 2.1) ('N133', 1.8, --) ('N115', 8.3, 2.5)],\n",
" mask = [(False, False, False) (False, False, True) (False, False, False)],\n",
" fill_value = ('N/A', 1e+20, 1e+20),\n",
" dtype = [('KeyName', 'S4'), ('cfloat1', '<f8'), ('cfloat2', '<f8')])"
]
}
],
"prompt_number": 10
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"j = join(f1, f2, join_type='outer', keys='KeyName')\n",
"j2 = join(j, f3, join_type='outer', keys='KeyName')\n",
"j2"
],
"language": "python",
"metadata": {},
"outputs": [
{
"html": [
"<table id=\"table4393064144\"><thead><tr><th>astr1</th><th>KeyName</th><th>KeyName2_1</th><th>bStr</th><th>KeyName2_2</th><th>cfloat1</th><th>cfloat2</th></tr></thead><tr><td>--</td><td>N111</td><td>--</td><td>CO5</td><td>jkl</td><td>--</td><td>--</td></tr><tr><td>--</td><td>N111</td><td>--</td><td>CO4</td><td>ghi</td><td>--</td><td>--</td></tr><tr><td>A219</td><td>N115</td><td>ghi</td><td>WN4</td><td>mno</td><td>8.3</td><td>2.5</td></tr><tr><td>A219</td><td>N115</td><td>def</td><td>WN4</td><td>mno</td><td>8.3</td><td>2.5</td></tr><tr><td>A02</td><td>N121</td><td>abc</td><td>--</td><td>--</td><td>--</td><td>--</td></tr><tr><td>--</td><td>N131</td><td>--</td><td>--</td><td>--</td><td>8.2</td><td>2.1</td></tr><tr><td>--</td><td>N133</td><td>--</td><td>--</td><td>--</td><td>1.8</td><td>--</td></tr></table>"
],
"metadata": {},
"output_type": "pyout",
"prompt_number": 11,
"text": [
"<Table rows=7 names=('astr1','KeyName','KeyName2_1','bStr','KeyName2_2','cfloat1','cfloat2')>\n",
"masked_array(data = [(--, 'N111', --, 'CO5', 'jkl', --, --)\n",
" (--, 'N111', --, 'CO4', 'ghi', --, --)\n",
" ('A219', 'N115', 'ghi', 'WN4', 'mno', 8.3, 2.5)\n",
" ('A219', 'N115', 'def', 'WN4', 'mno', 8.3, 2.5)\n",
" ('A02', 'N121', 'abc', --, --, --, --) (--, 'N131', --, --, --, 8.2, 2.1)\n",
" (--, 'N133', --, --, --, 1.8, --)],\n",
" mask = [(True, False, True, False, False, True, True)\n",
" (True, False, True, False, False, True, True)\n",
" (False, False, False, False, False, False, False)\n",
" (False, False, False, False, False, False, False)\n",
" (False, False, False, True, True, True, True)\n",
" (True, False, True, True, True, False, False)\n",
" (True, False, True, True, True, False, True)],\n",
" fill_value = ('N/A', 'N/A', 'N/A', 'N/A', 'N/A', 1e+20, 1e+20),\n",
" dtype = [('astr1', 'S4'), ('KeyName', 'S4'), ('KeyName2_1', 'S3'), ('bStr', 'S3'), ('KeyName2_2', 'S3'), ('cfloat1', '<f8'), ('cfloat2', '<f8')])"
]
}
],
"prompt_number": 11
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment