CSV Parsing in Python, Ruby and Golang

README.md

CSV Parsing in Python, Ruby and Golang

• https://github.com/zaiste/csv-playground
• https://zaiste.net/posts/csv_parsing_performance_python_vs_ruby/

data.csv

generator.rb

require 'csv'
require 'faker'

headers = %w(name email title image phone imei color company date ip url state country lat long card)

REPETITION = 10

names = Array.new(REPETITION){ |i| Faker::Name.name }
emails = Array.new(REPETITION){ |i| Faker::Internet.email }
titles = Array.new(REPETITION){ |i| Faker::Name.title }
images = Array.new(REPETITION){ |i| Faker::Avatar.image }
phones = Array.new(REPETITION){ |i| Faker::PhoneNumber.cell_phone }
imeis = Array.new(REPETITION){ |i| Faker::Code.imei }
colors = Array.new(REPETITION){ |i| Faker::Color.color_name }
companies = Array.new(REPETITION){ |i| Faker::Company.name }
dates = Array.new(REPETITION){ |i| Faker::Date.between(Date.new(2010,11,01), Date.today) }
ips = Array.new(REPETITION){ |i| Faker::Internet.ip_v4_address }
urls = Array.new(REPETITION){ |i| Faker::Internet.url }
states = Array.new(REPETITION){ |i| Faker::Address.state }
countries = Array.new(REPETITION){ |i| Faker::Address.country }
lats = Array.new(REPETITION){ |i| Faker::Address.latitude }
longs = Array.new(REPETITION){ |i| Faker::Address.longitude }
cards = Array.new(REPETITION){ |i| Faker::Business.credit_card_number }


CSV.open('data.csv', 'w', write_headers: true, headers: headers) do |csv|
  1_000_000.times do |i|
    csv << [
      names[i % REPETITION],
      emails[i % REPETITION],
      titles[i % REPETITION],
      images[i % REPETITION],
      phones[i % REPETITION],
      imeis[i % REPETITION],
      colors[i % REPETITION],
      companies[i % REPETITION],
      dates[i % REPETITION],
      ips[i % REPETITION],
      urls[i % REPETITION],
      states[i % REPETITION],
      countries[i % REPETITION],
      lats[i % REPETITION],
      longs[i % REPETITION],
      cards[i % REPETITION],
    ]
  end
end

reader.go

package main

import (
	"encoding/csv"
	"fmt"
  "os"
  "io"
)

func main() {
  file, err := os.Open("data-7M.csv")
	if err != nil {
		fmt.Println("Error:", err)
		return
	}
	defer file.Close()

	reader := csv.NewReader(file)
	for {
		_, err := reader.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Error:", err)
			return
		}
	}
}

reader.py

import time
import functools
import os
import csv
import progressbar
import psutil

csv_path = 'data.csv'

examples = []

def example(fn):
    @functools.wraps(fn)
    def wrapped():
        try:
            print('Running: {}'.format(fn.__name__))

            start = time.time()

            fn()

            end = time.time()
            print('{:.2f}'.format(end - start))

            process = psutil.Process(os.getpid())
            print('{:.2f} MB\n'.format(process.memory_info().rss / 1024 / 1024))
        except KeyboardInterrupt:
            print('Skipping...')

    examples.append(wrapped)
    return wrapped

@example
def basic():
    with open(csv_path, newline='') as f:
        reader = csv.reader(f)
        for row in reader:
            pass

@example
def with_progressbar():
    num_lines = sum(1 for line in open(csv_path))

    with progressbar.ProgressBar(max_value=num_lines) as bar:
        with open(csv_path, newline='') as f:
            reader = csv.reader(f)
            for row in bar(reader):
                pass

@example
def with_headers():
    with open(csv_path, newline='') as f:
        reader = csv.DictReader(f)
        for row in reader:
            pass


#import resource
#rint('{:.2f} MB'.format(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 / 1024))

for example in examples:
    example()

reader.rb

require 'csv'
require 'benchmark'
require 'ruby-progressbar'

FILENAME = 'data.csv'

def memory_usage
  memory_before = `ps -o rss= -p #{Process.pid}`.to_i
  yield
  memory_after = `ps -o rss= -p #{Process.pid}`.to_i

  puts "Memory: #{((memory_after - memory_before) / 1024.0).round(2)} MB"
end

def time_spent
  time = Benchmark.realtime do
    yield
  end

  puts "Time: #{time.round(2)}"
end

memory_usage do
  time_spent do
    count = File.foreach(FILENAME).inject(0) { |c, _| c + 1 }
    bar = ProgressBar.create(throttle_rate: 0.2, total: count)

    CSV.foreach(FILENAME, headers: true) do |row|
      bar.increment
    end
  end
end

writer.py

import csv

with open('eggs.csv', 'w', newline='') as csvfile:
    spamwriter = csv.writer(csvfile)
    spamwriter.writerow(['Spam'] * 5 + ['Baked Beans'])
    spamwriter.writerow(['Spam', 'Lovely, Spam', 'Wonderful Spam'])