many-many-queries-API.rb

mmqueries.rb

# queries.rb
queries = MMQueries.new do |config|
  # Scope: Receiver is <MMQueries> here.

  # add simple query and store the result
  add :top_n_keyword_of_day do |date|
    # Scope: Receiver is <MMQueries::Query> here.
    td.query('SELECT * FROM search WHERE ...'){|result|
      mongo.collection('search.keyword').insert(result.merge(date: date.to_time))
    } # This block is a callback called at completion of the job
  end
  
  # add a query including two job
  add :count_users_access do |date|
    %w[customers creators].each do |user_type|
      # Note: This two jobs are processed paralelly.
      #       Each callbacks are called asynchronously at completion of the job.
      td.query "SELECT * FROM access WHERE user_type = #{user_type}" {|result|
        mongo.collection('collectionA').insert(cnt: result[0], type: uesrs) 
      }
    end
  end
 
  # use `after` hook to store the accumulated result
  add :count_users_inflow_source do |range|
    range ||= Date.today.prev_day..Date.today # default argument
    result = %w[customers creators].each_with_object(Array.new) do |users,result|
      td.query "SELECT * FROM #{users} WHERE #{specific_time(from,to)}" {|partial_result|
        result << partial_result
      }
    end
    
    after do # this will called when all jobs have finished.
      mongo.collection('collectionA').insert(time: from, cnts: results.map(&:first))
    end
  end.every(1.day, :at => '4:30 am') # Assign schedule. This will be passed to whenever(gem) and be scheduled on crontab.
  
  # To throw a big query, here document is useful.
  add :count_users_inflow_source do |from, to|
    %w[customers creators].each do |users|
      td.query <<-QUERY {|result|
        SELECT * FROM #{users} WHERE ...
        JOIN
        another_table b
        ON (a.t1=b.t2)
      QUERY
        mongo.collection('collectionA').insert(cnt: result[0], type: uesrs) 
      }
    end
  end
 
   # grouping
  config.import_path = File.dirname(__FILE__) + '/queries'
  import 'kpi' # This import an object <MMQueries::Group> which is result of `eval File.read(config.import_path + 'kpi.rb')`
  import 'search/photo'
  
end


# queries/kpi.rb
MMQueries::Group.new do
  # Scope: Receiver is <MMQueries::Group> here.
  
  add :kpi_of_new_costomers do |..|
    ..
  end
  
  add :kpi_of_new_creators do |date|
    ..
    export result[0],date # `export` sets the arguments as result of the query.
  end
  
  # use `after` hook to store all result.
  after do |results|
    # result is Hash<query_label, job_result>
    results.each do |label,job_result|
      mongo.collection('all').insert({label => job_result})
    end
  end
end


# add some helper methods
module MMQueries::Helper # This module is included by MMQueries.
  def mongo
    return Mongo::Connection.new ..
  end
end
# There are three helper modules named `MMQueries::Helper`, `MMQueries::Group::Helper` and `MMQueries::Query::Helper`



### methods ###
## interface of MMQueries
queries.class # => MMQueries
queries.queries # => Hash<query_name,MMQueries::Query>
query = queries.look_up(:top_n_keyword_of_day) # => <MMQueries::Query>
group = queries.look_up_group(:kpi) # => <MMQueries::Group>

## Common interface with MMQueries, MMQuereis::Group, MMQueries::Query
queries.exec! # ... execute all jobs ... => all result
group.exec! # ... execute jobs ... => result
query.exec! # ... execute jobs ... => result

query.result # => result

query.exec # => nil # job and callback hook will be processed background.
query.wait! # wait until processing jobs have finished.

query.exec!(pry: true) # result.pry (inspection)

## only MMQueries::Query
query.exec!(Date.today) # ... execute jobs ... => result