Download 8-K filings from the SEC

download_8k_indexes.rb

require 'posix/spawn'
require 'pry'
require 'active_support/all'


years = (2010..2016).to_a
quarters = (1..4).to_a



years.product(quarters).each do |year, quarter|

        puts "Year: #{year} quarter #{quarter}"

        save_name = "Y#{year}Q#{quarter}-form.idx"
        file_uri = "ftp://ftp.sec.gov/edgar/full-index/#{year}/QTR#{quarter}/form.idx"
        pid = POSIX::Spawn::spawn("wget #{file_uri} -O #{save_name}")
        stat = Process::waitpid(pid)
        form_lines = POSIX::Spawn::Child.new("cat #{save_name} | grep ^8-K").out.split("\n")
        puts "Found #{form_lines.size} files for #{year}-#{quarter}"
        sleep 5
        form_lines.each_with_index do |line, line_index|


                puts "#{line_index}/#{form_lines.size} - #{save_name}"

                actual_document_url = line.split(/\s+/).last
                filing_date = line.split(/\s+/)[-2]
                local_filename = actual_document_url.split('/').last

                # maybe we already downloaded it before
                if File.exist?(local_filename) && File.size(local_filename).to_f > 0
                        puts "Skipping; already downloaded"
                else

                        file_uri = "ftp://ftp.sec.gov/#{actual_document_url}"
                        system("wget #{file_uri}")

                        GC.start
                end

        end


end