Dask Snippet

dask_snippet.py

##
# This method is quite slow for a large number of files (GBs)
# Just wanted to save some parts of the entire snippet for reference
# of some useful stuff like picking a row from a Dask dataframe
# and appending it to a Pandas Dataframe.

for id in unique_ids:
    # select all the rows across the partitions for the specified ID
    subset = catalogue[catalogue.id == id]
    # select only the one row with the latest date and time (max timestamp)
    subset = subset[subset.timestamp == subset.timestamp.max()]
    # append the selected row to the final dataset
    final_catalogue.loc[final_catalogue.shape[0]] = list(subset.to_dask_array().compute())[0]