Crawl a site and update Google Sheet with results

crawler

require 'spidr' # gem install spidr
require 'nokogiri'
require 'google_drive'

GOOGLE_SHEET_ID = "1k26clMVYUX5PLag5K2Ku4osgZcTtWp-wlss1B6CC_KE" # replace with your Google Sheet Id 
CRAWL_SITE = "https://www.mywebsite.com"

OpenSSL::SSL.send(:remove_const, :VERIFY_PEER)
OpenSSL::SSL.const_set(:VERIFY_PEER, OpenSSL::SSL::VERIFY_NONE)

@session = GoogleDrive.saved_session("config.json") # make sure to have your config.json file saved in the same folder

@ws = @session.spreadsheet_by_key(GOOGLE_SHEET_ID).worksheets[0]

@ws.title = "Results - Updated #{Time.now.strftime("%m/%d/%Y")}" # updates sheet title
@ws.save rescue nil

Spidr.site(CRAWL_SITE) do |spider|
  @idx = 0
  spider.every_page { |page|
    @idx = @idx + 1
    page.search('//meta').each do |meta|
      name = (meta.attributes['name'] || meta.attributes['http-equiv'])
          value = meta.attributes['content']
      case name.to_s
      when 'description'
        info = {
          :url => page.url.to_s,
          :title => page.search("title").text,
          :description => value,
          :status => page.code
        }
        row = 2 + @idx

        unless info.nil?
          @ws[row,1] = info[:url] || ''
          @ws[row,2] = info[:title] || ''
          @ws[row,3] = info[:description] || ''
          @ws[row,4] = info[:status] || ''
          @ws.save
        end
      else
      end
    end
  }
end

crawler.rb

require 'spidr' # gem install spidr
require 'nokogiri'
require 'google_drive'

GOOGLE_SHEET_ID = "1k26clMVYUX5PLag5K2Ku4osgZcTtWp-wlss1B6CC_KE" # replace with your Google Sheet Id 
CRAWL_SITE = "https://www.mywebsite.com"

OpenSSL::SSL.send(:remove_const, :VERIFY_PEER)
OpenSSL::SSL.const_set(:VERIFY_PEER, OpenSSL::SSL::VERIFY_NONE)

@session = GoogleDrive.saved_session("config.json") # make sure to have your config.json file saved in the same folder

@ws = @session.spreadsheet_by_key(GOOGLE_SHEET_ID).worksheets[0]

@ws.title = "Results - Updated #{Time.now.strftime("%m/%d/%Y")}" # updates sheet title
@ws.save rescue nil

Spidr.site(CRAWL_SITE) do |spider|
  @idx = 0
  spider.every_page { |page|
    @idx = @idx + 1
    page.search('//meta').each do |meta|
      name = (meta.attributes['name'] || meta.attributes['http-equiv'])
          value = meta.attributes['content']
      case name.to_s
      when 'description'
        info = {
          :url => page.url.to_s,
          :title => page.search("title").text,
          :description => value,
          :status => page.code
        }
        row = 2 + @idx

        unless info.nil?
          @ws[row,1] = info[:url] || ''
          @ws[row,2] = info[:title] || ''
          @ws[row,3] = info[:description] || ''
          @ws[row,4] = info[:status] || ''
          @ws.save
        end
      else
      end
    end
  }
end