Skip to content

Instantly share code, notes, and snippets.

@hhatto
Last active January 28, 2018 07:26
Show Gist options
  • Save hhatto/ce18792c3817c1ad67a7d4d8fe09ef5f to your computer and use it in GitHub Desktop.
Save hhatto/ce18792c3817c1ad67a7d4d8fe09ef5f to your computer and use it in GitHub Desktop.
benchmark script for python csv module
# coding: utf-8
from benchmarker import Benchmarker
import unicodecsv
import csv
import fcsv
source = [["abc", "def", "ghi"],
["jkl", "あいう", "opq"],
["vvv", "v1", "v2"]]
source = (("abc", "def", "ghi"),
("jkl", "あいう", "opq"),
("vvv", "v1", "v2"))
with Benchmarker(1000*1000, width=40) as bench:
@bench("std.writerows")
def b_std_writerows(bm):
with open('b_std.csv', 'w') as out:
writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
for i in bm:
writer.writerows(source)
@bench("std.writerow")
def b_std_writerow(bm):
with open('b_std.csv', 'w') as out:
writer = csv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
for i in bm:
writer.writerow(source[0])
writer.writerow(source[1])
writer.writerow(source[2])
@bench("unicodecsv.writerows")
def b_unicodecsv_writerows(bm):
with open('b_unicodecsv.csv', 'wb') as out:
writer = unicodecsv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
for i in bm:
writer.writerows(source)
@bench("unicodecsv.writerow")
def b_unicodecsv_writerow(bm):
with open('b_unicodecsv.csv', 'wb') as out:
writer = unicodecsv.writer(out, quoting=csv.QUOTE_NONNUMERIC)
for i in bm:
writer.writerow(source[0])
writer.writerow(source[1])
writer.writerow(source[2])
@bench("fcsv.writerow")
def b_fcsv_writerow(bm):
writer = fcsv.Writer('b_fcsv.csv') # , quoting=csv.QUOTE_NONNUMERIC)
for i in bm:
writer.writerow(source[0])
writer.writerow(source[1])
writer.writerow(source[2])
@bench("fcsv.writerows")
def b_fcsv_writerows(bm):
writer = fcsv.Writer('b_fcsv.csv') # , quoting=csv.QUOTE_NONNUMERIC)
for i in bm:
writer.writerows(source)
with Benchmarker(1000*1000, width=40) as bench:
@bench("std.reader")
def b_std_reader(bm):
with open('b_reader.csv') as fl:
reader = csv.reader(fl)
for i in bm:
for row in reader:
_ = row
@bench("unicodecsv.reader")
def b_unicodecsv_reader(bm):
with open('b_reader.csv', 'rb') as out:
reader = unicodecsv.reader(out, 'excel')
for i in bm:
for row in reader:
_ = row
@bench("fcsv.reader")
def b_fcsv_reader(bm):
reader = fcsv.Reader('b_reader.csv')
for i in bm:
for row in reader.read():
_ = row
@hhatto
Copy link
Author

hhatto commented Jan 24, 2018

## benchmarker:         release 4.0.1 (for python)
## python version:      3.6.4
## python compiler:     GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)
## python platform:     Darwin-16.7.0-x86_64-i386-64bit
## python executable:   /Users/hattori-h/.virtualenvs/py3csvbench/bin/python
## cpu model:           Intel(R) Core(TM) i5-5257U CPU @ 2.70GHz
## parameters:          loop=1000000, cycle=1, extra=0

##                                            real    (total    = user    + sys)
std.writerows                               2.2432    2.2200    2.1400    0.0800
std.writerow                                2.5543    2.5200    2.3700    0.1500
unicodecsv.writerows                        4.3601    4.3300    4.2000    0.1300
unicodecsv.writerow                         5.1060    5.0800    4.9000    0.1800
fcsv.writerow                               2.6175    2.6000    2.4600    0.1400
fcsv.writerows                              2.4747    2.4500    2.3300    0.1200

## Ranking                                    real
std.writerows                               2.2432  (100.0) ********************
fcsv.writerows                              2.4747  ( 90.6) ******************
std.writerow                                2.5543  ( 87.8) ******************
fcsv.writerow                               2.6175  ( 85.7) *****************
unicodecsv.writerows                        4.3601  ( 51.4) **********
unicodecsv.writerow                         5.1060  ( 43.9) *********

## Matrix                                     real    [01]    [02]    [03]    [04]    [05]    [06]
[01] std.writerows                          2.2432   100.0   110.3   113.9   116.7   194.4   227.6
[02] fcsv.writerows                         2.4747    90.6   100.0   103.2   105.8   176.2   206.3
[03] std.writerow                           2.5543    87.8    96.9   100.0   102.5   170.7   199.9
[04] fcsv.writerow                          2.6175    85.7    94.5    97.6   100.0   166.6   195.1
[05] unicodecsv.writerows                   4.3601    51.4    56.8    58.6    60.0   100.0   117.1
[06] unicodecsv.writerow                    5.1060    43.9    48.5    50.0    51.3    85.4   100.0

@hhatto
Copy link
Author

hhatto commented Jan 28, 2018

## benchmarker:         release 4.0.1 (for python)
## python version:      3.6.4
## python compiler:     GCC 4.2.1 Compatible Apple LLVM 9.0.0 (clang-900.0.39.2)
## python platform:     Darwin-16.7.0-x86_64-i386-64bit
## python executable:   /Users/hattori-h/.virtualenvs/py3csvbench/bin/python
## cpu model:           Intel(R) Core(TM) i5-5257U CPU @ 2.70GHz
## parameters:          loop=1000000, cycle=1, extra=0

##                                            real    (total    = user    + sys)
std.reader                                  3.8930    3.8900    3.3700    0.5200
unicodecsv.reader                           0.6580    0.6500    0.6500    0.0000
fcsv.reader                                 0.3270    0.3300    0.3300    0.0000

## Ranking                                    real
fcsv.reader                                 0.3270  (100.0) ********************
unicodecsv.reader                           0.6580  ( 49.7) **********
std.reader                                  3.8930  (  8.4) **

## Matrix                                     real    [01]    [02]    [03]
[01] fcsv.reader                            0.3270   100.0   201.2  1190.6
[02] unicodecsv.reader                      0.6580    49.7   100.0   591.7
[03] std.reader                             3.8930     8.4    16.9   100.0

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment