Downloading files from HDFS through zipping and Jupyterhub

gistfile1.txt

# Check the files are as expected
!hdfs dfs -ls /user/jorrick/my_file_path/

# Setup 
!mkdir /tmp/jorrick

# Create the subdirectory and make sure it has the correct permissions
!mkdir -m 700 /tmp/jorrick/my_file_path
!ls -ll /tmp/jorrick/my_file_path

# Copy the files over to local file system
!hdfs dfs -copyToLocal /user/jorrick/my_file_path/ /tmp/jorrick/my_file_path

# Zip everything, make sure you use enough stars :)
!zip /tmp/jorrick/my_file_path.zip /tmp/jorrick/my_file_path/*
!zip /tmp/jorrick/my_file_path.zip /tmp/jorrick/my_file_path/*/*
!zip /tmp/jorrick/my_file_path.zip /tmp/jorrick/my_file_path/*/*/*

# Verify your zip included the files by checking the file size
!ls -ll /tmp/jorrick/my_file_path.zip

# Upload the file back onto HDFS so we can then download it through the Jupyterhub interface
!hdfs dfs -put /tmp/jorrick/my_file_path.zip /user/jorrick/notebooks/

# Delete the tmp folder
!rm -rf /tmp/jorrick