Created
November 4, 2014 17:42
-
-
Save chiensiTB/0b3b20f2ba6f09962a30 to your computer and use it in GitHub Desktop.
uploading worldweather data to Ruby on Rails, without permissions failure on Heroku
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class CreatePlaces < ActiveRecord::Migration | |
# Rails migration for GeoWorldMap db into postgresql db | |
#(inspired by http://blog.inspired.no/populate-your-database-with-free-world-cities-countries-regions-in-2-minutes-using-a-rails-migration-273/ post) | |
#modified from the original gist, because of Heroku errors due to copy | |
#thanks to this post for helping me out: | |
#http://www.kadrmasconcepts.com/blog/2013/12/15/copy-millions-of-rows-to-postgresql-with-rails/ | |
#extract files from GeoWorldMap.zip archive from here | |
# http://www.geobytes.com/GeoWorldMap.zip | |
# | |
#and place them into #{Rails.root}/db/migrate/ | |
##the archive has 'cities.txt' file, rename it 'Cities.txt' | |
#mv cities.txt Cities.txt | |
def up | |
create_table :countries, :id => false do |t| | |
t.primary_key :CountryId | |
t.string :name, :limit => 50, :null => false | |
t.string :fips104, :limit => 2, :null => false | |
t.string :iso2, :limit => 2, :null => false | |
t.string :iso3, :limit => 3, :null => false | |
t.string :ison, :limit => 4, :null => false | |
t.string :internet, :limit => 2, :null => false | |
t.string :capital, :limit => 25 | |
t.string :map_reference, :limit => 50 | |
t.string :nationality_singular, :limit => 35 | |
t.string :nationality_plural, :limit => 35 | |
t.string :currency, :limit => 30 | |
t.string :currency_code, :limit => 3 | |
t.integer :population | |
t.string :title, :limit => 50 | |
t.string :comment, :limit => 255 | |
end | |
create_table :regions, :id => false do |t| | |
t.primary_key :RegionId | |
t.references :country, :null => false | |
t.string :name, :limit => 45, :null => false | |
t.string :code, :limit => 8, :null => false | |
t.string :adm1code, :limit => 4, :null => false | |
end | |
create_table :cities, :id => false do |t| | |
t.primary_key :CityId | |
t.references :country, :null => false | |
t.references :region, :null => false | |
t.string :name, :limit => 45, :null => false | |
t.float :latitude, :null => false | |
t.float :longitude, :null => false | |
t.string :timezone, :limit => 10, :null => false | |
t.integer :dma_id | |
t.string :code, :limit => 4 | |
end | |
add_index :countries, :name | |
add_index :regions, :name | |
add_index :cities, :name | |
#change encoding to the one used into files | |
execute "set client_encoding to 'latin1'" | |
conn = ActiveRecord::Base.connection | |
rc = conn.raw_connection | |
rc.exec("COPY countries FROM STDIN WITH CSV") | |
filepath = File.join(File.dirname(__FILE__),'Countries.txt') | |
file = File.open(filepath, 'r') | |
while !file.eof? | |
rc.put_copy_data(file.readline) | |
end | |
rc.put_copy_end | |
while res = rc.get_result | |
if e_message = res.error_message | |
p e_message | |
end | |
end | |
conn2 = ActiveRecord::Base.connection | |
rc2 = conn2.raw_connection | |
rc2.exec("COPY regions FROM STDIN WITH CSV") | |
filepath = File.join(File.dirname(__FILE__),'Regions.txt') | |
file2 = File.open(filepath, 'r') | |
while !file2.eof? | |
rc2.put_copy_data(file2.readline) | |
end | |
rc2.put_copy_end | |
while res = rc2.get_result | |
if e_message = res.error_message | |
p e_message | |
end | |
end | |
conn3 = ActiveRecord::Base.connection | |
rc3 = conn3.raw_connection | |
rc3.exec("COPY cities FROM STDIN WITH CSV") | |
filepath = File.join(File.dirname(__FILE__),'cities.txt') | |
file3 = File.open(filepath, 'r') | |
while !file3.eof? | |
rc3.put_copy_data(file3.readline) | |
end | |
rc3.put_copy_end | |
while res = rc3.get_result | |
if e_message = res.error_message | |
p e_message | |
end | |
end | |
#change encoding back to UTF8 | |
execute "set client_encoding to 'UTF8'" | |
rename_column :countries, 'CountryId', :id | |
rename_column :regions, 'RegionId', :id | |
rename_column :cities, 'CityId', :id | |
#to generate foreign keys add | |
#gem 'foreigner' into Gemfile | |
add_foreign_key(:cities, :countries, :dependent => :delete) | |
add_foreign_key(:cities, :regions, :dependent => :delete) | |
add_foreign_key(:regions, :countries, :dependent => :delete) | |
end | |
def down | |
drop_table :cities | |
drop_table :regions | |
drop_table :countries | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment