requester.rb

require 'rest-client'
require 'json'
require 'yaml'
require 'amazing_print'
require 'erb'
require 'csv'

requests = YAML.load_file('requester.yml')

def render(template, vars)
  ERB.new(template).result(
    OpenStruct.new(vars).instance_eval { binding }
  )
end

def parse_url(url, params)
  return url unless params

  params.each do |k, v|
    url.gsub!("{#{k}}", v.to_s)
  end

  url
end

def do_request(arguments = {})
  args = {
    url: parse_url(arguments['url'], arguments['url_params']),
    method: arguments['method'],
    headers: arguments['headers'] || {}
  }
  if arguments['method'].to_s == 'get'
    args[:headers][:params] = arguments['payload']
  else
    args[:payload] = arguments['payload']
  end
  res = RestClient::Request.execute(args)
  json = JSON.parse(res)

  response_key = arguments['response_key']
  return json unless response_key

  json.dig(*response_key.split('.'))
end

def export(request, data)
  export_hash = request['export']
  return unless export_hash

  type = export_hash['type']
  filename = export_hash['filename']
  headers = export_hash['headers']
  keys = export_hash['keys']

  return unless type == 'csv'

  CSV.open("./#{filename}.#{type}", "wb", col_sep: "\t", force_quotes: false) do |csv|
    csv << headers

    data.each do |row|
      values = []
      keys.each do |key|
        values << row.dig(*key.split('.'))
      end
      csv << values
    end
  end
end

requests.each do |request|
  data = do_request(request)
  export(request, data)
  ap '==========================================================='
end

requester.yml

-
  url: "http://localhost:1010/users"
  method: "get"
  headers:
  payload:
  url_params:
  response_key:
  export:
    type: "csv"
    filename: "users"
    headers:
      - "ID"
      - "Name"
      - "Email"
      - "City"
      - "Zip Code"
      - "Company"
    keys:
      - "id"
      - "name"
      - "email"
      - "address.city"
      - "address.zipcode"
      - "company.name"
-
  url: "http://localhost:1010/users/data"
  method: "get"
  headers:
  payload:
  url_params:
  response_key: "data"
  export:
    type: "csv"
    filename: "users_data"
    headers:
      - "ID"
      - "Name"
      - "Email"
      - "City"
      - "Zip Code"
      - "Company"
    keys:
      - "id"
      - "name"
      - "email"
      - "address.city"
      - "address.zipcode"
      - "company.name"
# -
#   url: "https://jsonplaceholder.typicode.com/posts/{id}"
#   method: "get"
#   headers:
#   payload:
#   url_params:
#     id: 1