Skip to content

Instantly share code, notes, and snippets.

@shiguruikai
Created July 12, 2020 11:38
Show Gist options
  • Save shiguruikai/20a13b725a620c7908cbd880682876b2 to your computer and use it in GitHub Desktop.
Save shiguruikai/20a13b725a620c7908cbd880682876b2 to your computer and use it in GitHub Desktop.
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
// find . -type f -print0 | xargs -0 -n 1 -P 24 md5sum | sort -t ' ' -k 2
public class GetFileHash {
static final int BUFFER_SIZE = 1024 * 8;
static final char[] HEX_ARRAY = "0123456789abcdef".toCharArray();
static String toHexString(byte[] bytes) {
char[] chars = new char[bytes.length * 2];
for (int i = 0; i < bytes.length; i++) {
int v = bytes[i] & 0xFF;
chars[i * 2] = HEX_ARRAY[v >>> 4];
chars[i * 2 + 1] = HEX_ARRAY[v & 0x0F];
}
return new String(chars);
}
@SuppressWarnings("StatementWithEmptyBody")
static String getFileHash(File file, MessageDigest md) throws IOException {
if (!file.isFile()) return null;
try (DigestInputStream dis = new DigestInputStream(Files.newInputStream(file.toPath(), StandardOpenOption.READ), md)) {
byte[] buf = new byte[BUFFER_SIZE];
while (dis.read(buf) != -1) {
}
}
return toHexString(md.digest());
}
public static void main(String[] args) throws Exception {
String algorithm = args[0];
String path = args[1];
MessageDigest messageDigest = MessageDigest.getInstance(algorithm);
Path rootPath = Paths.get(path).toAbsolutePath();
List<File> files = new ArrayList<>();
Files.walkFileTree(rootPath, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
files.add(file.toFile());
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) {
exc.printStackTrace(System.err);
return FileVisitResult.SKIP_SUBTREE;
}
});
files.parallelStream()
.map(file -> {
try {
String hash = getFileHash(file, (MessageDigest) messageDigest.clone());
String absolutePath = file.getAbsolutePath();
String relativePath = rootPath.relativize(file.toPath()).toString();
return new Item(hash, absolutePath, relativePath);
} catch (IOException e) {
e.printStackTrace(System.err);
return null;
} catch (Exception e) {
throw new RuntimeException(e);
}
})
.filter(Objects::nonNull)
.sorted(Comparator.comparing(Item::getRelativePath))
.collect(Collectors.toList())
.forEach(item -> System.out.println(item.getHash() + ' ' + item.getRelativePath()));
}
static class Item {
private final String hash;
private final String absolutePath;
private final String relativePath;
public Item(String hash, String absolutePath, String relativePath) {
this.hash = hash;
this.absolutePath = absolutePath;
this.relativePath = relativePath;
}
public String getHash() {
return hash;
}
public String getAbsolutePath() {
return absolutePath;
}
public String getRelativePath() {
return relativePath;
}
@Override
public String toString() {
return "Item{" +
"hash='" + hash + '\'' +
", absolutePath='" + absolutePath + '\'' +
", relativePath='" + relativePath + '\'' +
'}';
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment