Last active
March 8, 2019 08:29
-
-
Save dansku/13249d6f7f6427a365fcc160d464b283 to your computer and use it in GitHub Desktop.
organizing long list of txt files
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| DIR=/home/parse/big_lists2/ | |
| i=1 | |
| for fullfile in $DIR*.txt; do | |
| filename=$(basename -- "$fullfile") | |
| extension="${filename##*.}" | |
| filename="${filename%.*}" | |
| hash=$(md5sum "${fullfile}"|cut -d' ' -f1) | |
| echo "Moving" $filename".txt to "$filename | |
| echo "[$i] Making directory folder_$i" | |
| mkdir $DIR"folder_"$i | |
| echo "[$i] Move "$filename".txt to folder_"$i" renamed to "$hash".txt" | |
| mv $filename".txt" $DIR"folder_"$i/$hash.txt | |
| echo "[$i] Enter dir "$i | |
| cd $DIR"folder_"$i | |
| echo "[$i] Split files" | |
| split -l 100000 --additional-suffix=.txt --numeric-suffixes $hash".txt" $i"_" | |
| echo "[$i] Delete hash file" | |
| rm $hash.txt | |
| echo "[$i] Move new files to root directory" | |
| mv *.txt .. | |
| echo "[$i] Deleting folder $i" | |
| cd .. && rm -rf folder_$i | |
| echo "----------------------------------------------------" | |
| i=$((i+1)) | |
| done | |
| #------------------ | |
| # Rename all files for its hash | |
| for file in *.txt; do hash=$(md5sum "${file}"|cut -d' ' -f1) && echo $file $hash.txt && mv $file $hash.txt; done | |
| #----------------- | |
| # Rename all hash to number |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment