save_it.rb

#!/usr/bin/env ruby
require 'zlib'
require 'open-uri'
require 'sqlite3'

class Torrent
  # hash, name, genre, link, torrent
  attr_accessor :name, :magnet, :link, :genre, :torrent
 
  def initialize(line)
    begin
      parts = line.split("|")
      @name = parts[1].strip
      @magnet = "magnet:?xt=urn:btih:#{parts[0].strip}"
      @genre = parts[2].strip
      @link = parts[3].strip
      @torrent = parts[4].strip
    rescue Exception => e
      print "Skipped: "
      puts line
    end
  end
end
 
def parse_torrents(link)
  open(link) do |f|
    #gz = Zlib::GzipReader.new(f)
    gz = open(f).read
    torrents = []
    gz.each_line do |line|
      next if line.strip.size == 0
      torrents << Torrent.new(line)
    end
    gz.close
    torrents
  end
end
 
def main
  begin  
    db = SQLite3::Database.new "data.db"
    db.execute "CREATE TABLE IF NOT EXISTS Torrents(id INTEGER PRIMARY KEY AUTOINCREMENT, 
        name TEXT, magnet TEXT, link TEXT, genre TEXT, torrent TEXT)"
  rescue SQLite3::Exception => e 
    puts "Exception occurred"
    puts e
  end

  # 6361784 dailydump.txt
  # https://kickass.to/hourlydump.txt.gz
  if ARGV.size == 1
    gz = ARGV.first
  else
    gz = "https://kickass.to/hourlydump.txt.gz"
  end
  Zlib::GzipReader.new(open gz).readlines.each_with_index do |line, index|
    next if line.strip.size == 0
    tr = Torrent.new(line)

    shouldSelect = true
    if shouldSelect
      stmt = db.prepare "SELECT * FROM Torrents WHERE magnet=?"
      stmt.bind_params tr.magnet
      rs = stmt.execute
      if rs.next
        puts "Skip...#{index}!"
        stmt.close
        next
      else
        shouldSelect = false
      end
      stmt.close
    end
    
    puts "Inserting...#{index}"
    stm = db.prepare "INSERT INTO Torrents (name, magnet, link, genre, torrent) VALUES(?, ?, ?, ?, ?)"
    stm.bind_params tr.name, tr.magnet, tr.link, tr.genre, tr.torrent
    stm.execute
    stm.close
  end
  db.close
end

main