TensorFlow Arrow Blog Part 9 - Read and Process Directory

tf_arrow_blog_pt9.py

def read_and_process_dir(directory):
  """Read a directory of CSV files and yield processed Arrow batches."""

  for f in os.listdir(directory):
    if f.endswith(".csv"):
      filename = os.path.join(directory, f)
      for batch in read_and_process(filename):
        yield batch