Scraping the DMR "Find Wells" Table

department_of_mineral_resources.rb

#Copyright 2018 Tariq R. Ali

#Licensed under the Apache License, Version 2.0 (the "License");
#you may not use this file except in compliance with the License.
#You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

#Unless required by applicable law or agreed to in writing, software
#distributed under the License is distributed on an "AS IS" BASIS,
#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#See the License for the specific language governing permissions and
#limitations under the License.

require 'watir' #code tested on watir-6.11.0, and uses Headless Chrome

f = File.open('government_data.csv', 'w')

def write_to_csv(text)
  open('government_data.csv', 'a') { |f|
    f.puts text
  }
end

browser = Watir::Browser.new :chrome, headless: true

browser.goto "https://www.dmr.nd.gov/oilgas/findwellsvw.asp"
form = browser.form

ddm_field = form.select_list id: 'ddmField'

fields = ddm_field.options.to_a[1..-1].map do |option|
  option.value
end

fields.each do |field|

  ddm_field.select field

  browser.forms[0].submit

  table_rows = browser.tables[1].trs.to_a[1..-1]

  table_rows.collect do |tr|
    csv_row = []

    0.upto(10) do |counter|
      column = tr[counter]
      csv_row << column.text
    end
    write_to_csv(csv_row.join('|'))
  end
end

puts "We have finished scraping the site."