AW-ETL configuration

database.yml

development:
  adapter: sqlite3
  database: db/development.sqlite3
  pool: 5
  timeout: 5000

test:
  adapter: sqlite3
  database: db/test.sqlite3
  pool: 5
  timeout: 5000

etl_execution:
  database: rails_warehouse_etl_execution
  adapter: mysql2
  hostname: localhost
  username: root
  password: root
  socket: /Applications/MAMP/tmp/mysql/mysql.sock

# Client databases
data_source_abc_development:
  adapter: pg
  database: abc
  hostname: localhost
  user: postgres
  password: postgres
                            

extract.ctl

# What is happening here:
#   1. Connects to PostgreSQL data source
#   2. Extracts data from articulos, and related info; checks for updated and new records
#   3. Transform and normalize records
#   4. Insert data into Tablios' MongoDB database.

ETL::Engine.logger = Logger.new(STDOUT)
ETL::Engine.logger.level = Logger::DEBUG

source :abc_development, {
  :type => :database,
  :target => :data_source_abc_development,
  :table => "articulos"
},
[
  :articulo,
  :titulo,
  :email
]

# For debugging
after_read do |row| ap row; row; end

before_write do |row|
  mongo_connection.insert(row)
  row
end

extract_extended.ctl

# What is happening here:
#   1. Connects to PostgreSQL data source
#   2. Extracts data from articulos, and related info; checks for updated and new records
#   3. Transforms and normalizes records
#   4. Inserts data into Tablios' MongoDB database.

ETL::Engine.logger = Logger.new(STDOUT)
ETL::Engine.logger.level = Logger::DEBUG

# raise "Missing required DB environment variable" unless ENV['DB']
database_yml = File.dirname(__FILE__) + '/config/database.yml'

# ETL::Engine.init(:config => database_yml)
# ActiveRecord::Base.establish_connection :data_souce_abc_development

# ActiveRecord::Base.establish_connection(
#   :adapter =>   "pg",
#   :database =>  "abc",
#   :host => "localhost",
#   :username => "postgres",
#   :password => "postgres",
# )
# ActiveRecord::Base.establish_connection(:data_souce_development)

# Extraction process
source :abc_development, {
  :type => :database,
  :target => :data_source_abc_development,
  :table => "articulos"
},
[
  :articulo,
  :titulo,
  :email
]

# For debugging
after_read do |row| ap row; row; end

before_write do |row|
  mongo_connection.insert(row)
  row
end