Skip to content

Instantly share code, notes, and snippets.

@wesm

wesm/repro.py

Created September 27, 2018 22:10
Show Gist options
  • Save wesm/56c66524b1237a08a528c6eb8d01e44b to your computer and use it in GitHub Desktop.
Save wesm/56c66524b1237a08a528c6eb8d01e44b to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 42, in read_fastparquet
pf = ParquetFile(SpyReader(f), open_with=spy_open)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 100, in __init__
self._parse_header(fn, verify)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 122, in _parse_header
f.seek(0)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(0, whence=0)
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 42, in read_fastparquet
pf = ParquetFile(SpyReader(f), open_with=spy_open)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 100, in __init__
self._parse_header(fn, verify)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 125, in _parse_header
f.seek(-8, 2)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(-8, whence=2)
Tried to read 4, got 4
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 42, in read_fastparquet
pf = ParquetFile(SpyReader(f), open_with=spy_open)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 100, in __init__
self._parse_header(fn, verify)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 132, in _parse_header
f.seek(-(head_size+8), 2)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(-826, whence=2)
Tried to read 65536, got 826
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 42, in read_fastparquet
pf = ParquetFile(SpyReader(f), open_with=spy_open)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 100, in __init__
self._parse_header(fn, verify)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 134, in _parse_header
fmd = read_thrift(f, parquet_thrift.FileMetaData)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/thrift_structures.py", line 32, in read_thrift
file_obj.seek(starting_pos + blocks * bufsize + buffer_pos)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(459930, whence=0)
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 43, in read_fastparquet
return pf.to_pandas(['A'])
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 426, in to_pandas
index=index, assign=parts)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 258, in read_row_group
scheme=self.file_scheme)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 344, in read_row_group
cats, selfmade, assign=assign)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 321, in read_row_group_arrays
catdef=out.get(name+'-catdef', None))
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 194, in read_col
infile.seek(off)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(4, whence=0)
Tried to read 65536, got 65536
Tried to read 65536, got 65536
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 43, in read_fastparquet
return pf.to_pandas(['A'])
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 426, in to_pandas
index=index, assign=parts)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 258, in read_row_group
scheme=self.file_scheme)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 344, in read_row_group
cats, selfmade, assign=assign)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 321, in read_row_group_arrays
catdef=out.get(name+'-catdef', None))
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 195, in read_col
ph = read_thrift(infile, parquet_thrift.PageHeader)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/thrift_structures.py", line 32, in read_thrift
file_obj.seek(starting_pos + blocks * bufsize + buffer_pos)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(98300, whence=0)
Tried to read 163801, got 163801
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 43, in read_fastparquet
return pf.to_pandas(['A'])
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 426, in to_pandas
index=index, assign=parts)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 258, in read_row_group
scheme=self.file_scheme)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 344, in read_row_group
cats, selfmade, assign=assign)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 321, in read_row_group_arrays
catdef=out.get(name+'-catdef', None))
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 194, in read_col
infile.seek(off)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(229379, whence=0)
Tried to read 65536, got 65536
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 43, in read_fastparquet
return pf.to_pandas(['A'])
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 426, in to_pandas
index=index, assign=parts)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 258, in read_row_group
scheme=self.file_scheme)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 344, in read_row_group
cats, selfmade, assign=assign)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 321, in read_row_group_arrays
catdef=out.get(name+'-catdef', None))
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 195, in read_col
ph = read_thrift(infile, parquet_thrift.PageHeader)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/thrift_structures.py", line 32, in read_thrift
file_obj.seek(starting_pos + blocks * bufsize + buffer_pos)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(229420, whence=0)
Tried to read 46, got 46
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 43, in read_fastparquet
return pf.to_pandas(['A'])
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 426, in to_pandas
index=index, assign=parts)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 258, in read_row_group
scheme=self.file_scheme)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 344, in read_row_group
cats, selfmade, assign=assign)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 321, in read_row_group_arrays
catdef=out.get(name+'-catdef', None))
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 194, in read_col
infile.seek(off)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(229557, whence=0)
Tried to read 65536, got 65536
Tried to read 65536, got 65536
File "/home/wesm/miniconda/envs/arrow-dev/bin/ipython", line 11, in <module>
sys.exit(start_ipython())
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/__init__.py", line 125, in start_ipython
return launch_new_instance(argv=argv, **kwargs)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/ipapp.py", line 356, in start
self.shell.mainloop()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 485, in mainloop
self.interact()
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/terminal/interactiveshell.py", line 476, in interact
self.run_cell(code, store_history=True)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2909, in run_ast_nodes
if self.run_code(code, result):
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-66-37b4667983c2>", line 1, in <module>
read_fastparquet()
File "<ipython-input-65-9dec9f99efd8>", line 43, in read_fastparquet
return pf.to_pandas(['A'])
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 426, in to_pandas
index=index, assign=parts)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/api.py", line 258, in read_row_group
scheme=self.file_scheme)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 344, in read_row_group
cats, selfmade, assign=assign)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 321, in read_row_group_arrays
catdef=out.get(name+'-catdef', None))
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/core.py", line 195, in read_col
ph = read_thrift(infile, parquet_thrift.PageHeader)
File "/home/wesm/miniconda/envs/arrow-dev/lib/python3.6/site-packages/fastparquet/thrift_structures.py", line 32, in read_thrift
file_obj.seek(starting_pos + blocks * bufsize + buffer_pos)
File "<ipython-input-65-9dec9f99efd8>", line 31, in seek
print(''.join(traceback.format_stack()))
Seek(327853, whence=0)
Tried to read 163801, got 132085
Raw
repro.py
import os
import pandas as pd
import pyarrow
import pyarrow.parquet as arrow_pq
from fastparquet import ParquetFile
# data to generate
ROW_LENGTH = 40000 # decreasing below 32750ish eliminates exception
N_ROWS = 10
# file write params
ROW_GROUP_SIZE = 5 # Lower numbers eliminate exception, but strange data is read (e.g. Nones)
FILENAME = 'test.parquet'
def write_arrow(length):
df = pd.DataFrame({'A': ['A' * length for _ in range(N_ROWS)]})
if os.path.isfile(FILENAME):
os.remove(FILENAME)
arrow_table = pyarrow.Table.from_pandas(df)
arrow_pq.write_table(arrow_table,
FILENAME,
use_dictionary=False,
compression='NONE',
row_group_size=ROW_GROUP_SIZE)
def read_arrow():
print("arrow:")
table2 = arrow_pq.read_table(FILENAME)
return table2.to_pandas()
for i in range(30000, 40000):
print(i)
write_arrow(i)
df1 = read_arrow()
df2 = read_fastparquet()
pd.util.testing.assert_frame_equal(df1, df2)
import io
import traceback
class SpyReader(object):
def __init__(self, f):
self.f = f
def __del__(self):
return self.f.close()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, tb):
self.close()
def close(self):
return self.f.close()
def tell(self):
return self.f.tell()
def read(self, nbytes):
result = self.f.read(nbytes)
print('Tried to read {0}, got {1}'.format(nbytes, len(result)))
return result
def seek(self, position, whence=0):
print(''.join(traceback.format_stack()))
print('Seek({0}, whence={1})'.format(position, whence))
return self.f.seek(position, whence)
def spy_open(filename, mode='rb'):
return SpyReader(open(filename, mode))
def read_fastparquet():
with open(FILENAME, 'rb') as f:
pf = ParquetFile(SpyReader(f), open_with=spy_open)
return pf.to_pandas(['A'])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment