Last active
December 12, 2015 01:39
-
-
Save thejefflarson/4693128 to your computer and use it in GitHub Desktop.
These two examples are what I mean when I say learn to program. The first is the upper limits of knowledge, and the second is pretty much all an average journalist needs to know. If you can read the second you probably have all the knowledge a journalist needs. The bottom would take about a sum total of a week of study to learn. The top one take…
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'thread' | |
require 'csv' | |
require 'net/http' | |
require 'json' | |
id = 0 | |
max_id = 30 | |
class Atomic | |
def initialize(val) | |
@value = val | |
@m = Mutex.new | |
end | |
def get | |
@m.synchronize { @value } | |
end | |
def update | |
true and Thread.pass until compare_and_set old = get, new_value = yield(old) | |
new_value | |
end | |
def compare_and_set(old, new_val) | |
return false unless @m.try_lock | |
begin | |
return false unless @value.equal? old | |
@value = new_val | |
ensure | |
@m.unlock | |
end | |
true | |
end | |
end | |
def get(id) | |
uri = URI("https://api.github.com/legacy/repos/search/twitter?start_page=#{id}") | |
http = Net::HTTP.new(uri.host, uri.port) | |
http.use_ssl = true | |
http.verify_mode = OpenSSL::SSL::VERIFY_NONE | |
resp = http.get(uri.request_uri).body | |
JSON.parse(resp) | |
end | |
csv = CSV.open("./github.csv", "wb") | |
lock = Mutex.new | |
atomic = Atomic.new(id) | |
headers = get(1)['repositories'][0].keys | |
csv << headers | |
5.times.map { |i| | |
Thread.new do |t| | |
c = 0 | |
loop do | |
c += 1 | |
id = atomic.update { |v| v + 1 } | |
break if id >= max_id | |
json = get(id) | |
next unless json | |
arr = json['repositories'].map {|it| headers.map {|h| it[h] } } | |
lock.synchronize { csv << arr; puts "#{id}, #{i}: #{c}" } | |
end | |
end | |
}.map(&:join) | |
csv.close |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
require 'csv' | |
require 'net/http' | |
require 'json' | |
id = 0 | |
max_id = 30 | |
def get(id) | |
uri = URI("https://api.github.com/legacy/repos/search/twitter?start_page=#{id}") | |
http = Net::HTTP.new(uri.host, uri.port) | |
http.use_ssl = true | |
http.verify_mode = OpenSSL::SSL::VERIFY_NONE | |
resp = http.get(uri.request_uri).body | |
JSON.parse(resp) | |
end | |
csv = CSV.open("./github.csv", "wb") | |
headers = get(1)['repositories'][0].keys | |
csv << headers | |
id = 0 | |
loop do | |
id += 1 | |
break if id >= max_id | |
json = get(id) | |
next unless json | |
arr = json['repositories'].map {|it| headers.map {|h| it[h] } } | |
csv << arr | |
end | |
csv.close |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment