sha1-bug.rb

require 'digest/sha1'

def buf_sha file, buf_size
  d = Digest::SHA1.new
  buf = String.new

  open(file) do |io|
    d.update buf while io.read(buf_size, buf)
  end

  d
end

file = 'data'
MiB = 1024 ** 2

# uncomment the line below to generate the data file
# `dd if=/dev/urandom of=#{file} bs=#{MiB} count=550`

puts "Digest::SHA1#file:\t#{Digest::SHA1.file(file)}"

buf_size = 512 * MiB
puts "updated < 512 MiB:\t#{buf_sha(file, buf_size - 1)}"
puts "updated = 512 MiB:\t#{buf_sha(file, buf_size)}"
puts "updated = file size:\t#{buf_sha(file, File.size(file))}"

ed@curry:/735465$ sha1sum data
43a8c9e911fdbacf382f45287ef7e592672f184c data
ed@curry:/735465$ ruby sha1-bug.rb
550+0 records in
550+0 records out
576716800 bytes (577 MB) copied, 104.668 s, 5.5 MB/s
Digest::SHA1#file: 56a35cadf6e96329cade8ab027153d431c6e4d5c
updated < 512 MiB: 56a35cadf6e96329cade8ab027153d431c6e4d5c
updated = 512 MiB: 5b765e82c4813b0ada85f841269e8386a499b10e
updated = file size: 5b765e82c4813b0ada85f841269e8386a499b10e

color me confused

i think it might have something with mmap and the maximun size of an the number used for the offset, i never looked into it tho

On Tue, Sep 27, 2011 at 2:27 PM, Ed Summers < ***@***.***>wrote: color me confused ## Reply to this email directly or view it on GitHub: https://gist.github.com/735465

## franco

At any rate I've got a merge request for your bagit library which I think addresses large file support.