Dealing with large number of files

large_files.sh

# Quickly count, -f disable sorting, adding . and ..
ls -f directory | wc -l

# Quickly remove files
rsync -a --delete empty_directory/ directory/
find directory -delete

# Quickly copy files
rsync -a source/ target/
# Copy list of files
rsync -a --files-from=/path/to/file-list source target
# Replace scp
rsync -av --progress source remote:target/
rsync -av -e 'ssh -p port_number' source remote:target/

# Grep large files in parallel
parallel --pipe grep 'pattern' < bigfile.txt
parallel --pipe grep -f patterns.txt < bigfile.txt