Sample and Read CSV using Pandas

sample.py

"""Sample and read sample into pandas"""

import subprocess
import urllib

import pandas as pd

# City of Raleigh, NC Open Data -- Building Permits (2020-11-02)
URI = "https://opendata.arcgis.com/datasets/bdfad82b15344d37beb28d7f90b6c4be_0.csv"
FULL_FN = "full-dataset.csv"
SAMPLE_FN = "sample.csv"

urllib.urlretrieve(URI, FN)

bashCommand = f"shuf -n 100000 {FN} > {SAMPLE_FN}"
process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
output, error = process.communicate()

df = pd.read_csv(SAMPLE_FN)
print(df.head())

# Do processing here