eval-timeit.py

#!/usr/bin/env python3

import gzip
import timeit
import ast
import os
import time
import better_exchook
import json

better_exchook.install()
txt_fn_gz = os.path.expanduser("train-other-500.txt.gz")

t = time.time()
txt = gzip.open(txt_fn_gz, "rb").read().decode("utf8")
print("Gunzip + read time:", time.time() - t)
print("Size:", len(txt))

print("compile:", timeit.timeit(lambda: compile(txt, "<>", "exec"), number=1))
print("parse:", timeit.timeit(lambda: compile(txt, "<>", "exec", ast.PyCF_ONLY_AST), number=1))
print("eval:", timeit.timeit(lambda: eval(txt), number=1))
print("ast.literal_eval:", timeit.timeit(lambda: ast.literal_eval(txt), number=1))

content = eval(txt)
js = json.dumps(content)

print("json.loads:", timeit.timeit(lambda: json.loads(js), number=1))

generate-dummy-train-txt-gz.py

#!/usr/bin/env python3

import better_exchook
import gzip
import random

better_exchook.install()
rnd = random.Random(42)
chars = "abc def ghi jkl mno pqr stu vwx yz '\" ABC XY0123.,"

with gzip.open("train-other-500.txt.gz", "w") as f:
  f.write(b"[\n")
  for i in range(100000):
    txt_len = rnd.randint(5, 200)
    d = {
      "seq_index": i, "seq_name": "seq-%i" % i, 'file': "seq-%i.flac" % i,
      "text": "".join(rnd.choice(chars) for _ in range(txt_len)),
      "duration": rnd.uniform(1, 60)}
    f.write(b"%r,\n" % (d,))
  f.write(b"]\n")