fetch-bvmf-symbols.rb

require 'net/http'
require 'nokogiri'

$uri = URI('http://bvmf.bmfbovespa.com.br/cias-listadas/empresas-listadas/BuscaEmpresaListada.aspx?idioma=pt-br')

def fetch_html(post_data)
  req = Net::HTTP::Post.new($uri)
  req.body = post_data

  res = Net::HTTP.start($uri.hostname, $uri.port) { |http| http.request(req) }
  res.value unless res.kind_of?(Net::HTTPSuccess)

  Nokogiri::HTML(res.body)
end

def fetch_symbols_from_segment(segment)
  post_data = '__EVENTTARGET=ctl00%3AcontentPlaceHolderConteudo%3ABuscaSegmentoEmpresa1%3AcmbSegmento&__EVENTARGUMENT=&__VIEWSTATE=%2FwEPDwULLTE2OTg3NzU5NjgPZBYCZg9kFgQCAQ9kFgICDw8WAh4EVGV4dAUIbWVyY2Fkb3NkAgMPZBYCAgEPZBYEAgEPZBYCAgEPDxYCHgNfYWUFQmN0bDAwX2NvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvX3RhYk1lbnVFbXByZXNhTGlzdGFkYV90YWJTZWdtZW50b2QWBgIFDw8WAh8ABQoxMS8xMi8yMDE2ZGQCCQ8PFgIfAAUFMDVoMDdkZAIRDw8WBB4NU2VsZWN0ZWRJbmRleAICHgVDb3VudAIEZBYCAgMPDxYGHgJJRAUacHZ3QnVzY2FBbm9MaXN0YWdlbUVtcHJlc2EeB1Rvb2xUaXBlHgdWaXNpYmxlaGQWAgIBD2QWAgIBDxBkZBYAZAIDDw9kFgIeB29uY2xpY2sFHGhpc3RvcnkuYmFjaygpOyByZXR1cm4gZmFsc2VkGAYFPGN0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEJ1c2NhU2V0b3JFbXByZXNhMSRncmRTZXRvcg8UKwACZBcAZAVBY3RsMDAkY29udGVudFBsYWNlSG9sZGVyQ29udGV1ZG8kQnVzY2FTZWdtZW50b0VtcHJlc2ExJGdyZEVtcHJlc2EPFCsAAmQXAGQFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYIBT1jdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYU5vbWVFbXByZXNhMSRncmRFbXByZXNhBTxjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYVNldG9yRW1wcmVzYTEkZ3JkU2V0b3IFPmN0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEJ1c2NhU2V0b3JFbXByZXNhMSRncmRFbXByZXNhBUFjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYVNlZ21lbnRvRW1wcmVzYTEkZ3JkRW1wcmVzYQVEY3RsMDAkY29udGVudFBsYWNlSG9sZGVyQ29udGV1ZG8kQnVzY2FBbm9MaXN0YWdlbUVtcHJlc2ExJGdyZEVtcHJlc2EFL2N0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEFqYXhQYW5lbEJ1c2NhBStjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRtcGdQYWdpbmFzBTZjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyR0YWJNZW51RW1wcmVzYUxpc3RhZGEFRGN0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEJ1c2NhQW5vTGlzdGFnZW1FbXByZXNhMSRncmRFbXByZXNhDxQrAAJkFwBkBT5jdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYVNldG9yRW1wcmVzYTEkZ3JkRW1wcmVzYQ8UKwACZBcAZAU9Y3RsMDAkY29udGVudFBsYWNlSG9sZGVyQ29udGV1ZG8kQnVzY2FOb21lRW1wcmVzYTEkZ3JkRW1wcmVzYQ8UKwACZBcAZHGpxReMu9KWVnXhRv4gmMRHNHyo&__EVENTVALIDATION=%2FwEWDgK5m62xBALp2e%2BmCgLwq5u4DgLntrXLBgLntrnLBgLmtuXIBgLmtqHLBgLmtr3LBgLmtsXIBgLntoXLBgLmtrXLBgLR5fGCBAK6%2FNOfAgK0s%2BTyDhrwW67mNT8%2FHuuO3FS9WooEe%2B%2Bu&ctl00_contentPlaceHolderConteudo_AjaxPanelBuscaPostDataValue=ctl00_contentPlaceHolderConteudo_AjaxPanelBusca%2CActiveElement%2Cctl00_contentPlaceHolderConteudo_BuscaSegmentoEmpresa1_cmbSegmento%3B&ctl00$contentPlaceHolderConteudo$tabMenuEmpresaListada=%7B%22State%22%3A%7B%7D%2C%22TabState%22%3A%7B%22ctl00_contentPlaceHolderConteudo_tabMenuEmpresaListada_tabSegmento%22%3A%7B%22Selected%22%3Atrue%7D%7D%7D&ctl00_contentPlaceHolderConteudo_BuscaSegmentoEmpresa1_grdEmpresaPostDataValue=&ctl00$contentPlaceHolderConteudo$mpgPaginas_Selected=2&__LASTFOCUS=&RadAJAXControlID=ctl00_contentPlaceHolderConteudo_AjaxPanelBusca&httprequest=true'
  doc = fetch_html("#{post_data}&ctl00$contentPlaceHolderConteudo$BuscaSegmentoEmpresa1$cmbSegmento=#{segment}")

  table = doc.xpath('//table[@class="MasterTable_SiteBmfBovespa"]').first

  symbols = Hash.new { |h, k| h[k] = [] }

  table.xpath('.//td/a').each do |row|
    name = row.text
    code = row['href'].match(/=(.+?)$/)[1]
    url = "http://bvmf.bmfbovespa.com.br/pt-br/mercados/acoes/empresas/ExecutaAcaoConsultaInfoEmp.asp?CodCVM=#{code}&ViewDoc=0"

    res_company = Net::HTTP.get_response(URI(url))
    res_company.value unless res_company.kind_of?(Net::HTTPSuccess)

    doc_company = Nokogiri::HTML(res_company.body)
    doc_company.xpath('//a[@class="LinkCodNeg"]').each do |symbol|
      symbols[symbol.text] << name
    end

    STDERR.print(".")
  end
  STDERR.print("\n")

  symbols.keys.sort.each { |s| puts(s) }
end

def fetch_segments
  doc = fetch_html('__EVENTTARGET=ctl00%3AcontentPlaceHolderConteudo%3AtabMenuEmpresaListada&__EVENTARGUMENT=ctl00%24contentPlaceHolderConteudo%24tabMenuEmpresaListada%24tabSegmento&__VIEWSTATE=%2FwEPDwULLTE2OTg3NzU5NjgPZBYCZg9kFgQCAQ9kFgICEA8WAh4EVGV4dAUIbWVyY2Fkb3NkAgMPZBYCAgEPZBYEAgEPZBYCAgEPZBYGAgUPDxYCHwAFCjIwLzA3LzIwMTdkZAIJDw8WAh8ABQUwNWgxN2RkAhEPDxYCHgVDb3VudAIEZBYCAgMPDxYGHgJJRAUacHZ3QnVzY2FBbm9MaXN0YWdlbUVtcHJlc2EeB1Rvb2xUaXBlHgdWaXNpYmxlaGQWAgIBD2QWAgIBDxBkZBYAZAIDDw9kFgIeB29uY2xpY2sFHGhpc3RvcnkuYmFjaygpOyByZXR1cm4gZmFsc2VkGAYFPGN0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEJ1c2NhU2V0b3JFbXByZXNhMSRncmRTZXRvcg8UKwACZBcAZAVBY3RsMDAkY29udGVudFBsYWNlSG9sZGVyQ29udGV1ZG8kQnVzY2FTZWdtZW50b0VtcHJlc2ExJGdyZEVtcHJlc2EPFCsAAmQXAGQFHl9fQ29udHJvbHNSZXF1aXJlUG9zdEJhY2tLZXlfXxYIBT1jdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYU5vbWVFbXByZXNhMSRncmRFbXByZXNhBTxjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYVNldG9yRW1wcmVzYTEkZ3JkU2V0b3IFPmN0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEJ1c2NhU2V0b3JFbXByZXNhMSRncmRFbXByZXNhBUFjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYVNlZ21lbnRvRW1wcmVzYTEkZ3JkRW1wcmVzYQVEY3RsMDAkY29udGVudFBsYWNlSG9sZGVyQ29udGV1ZG8kQnVzY2FBbm9MaXN0YWdlbUVtcHJlc2ExJGdyZEVtcHJlc2EFL2N0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEFqYXhQYW5lbEJ1c2NhBStjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRtcGdQYWdpbmFzBTZjdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyR0YWJNZW51RW1wcmVzYUxpc3RhZGEFRGN0bDAwJGNvbnRlbnRQbGFjZUhvbGRlckNvbnRldWRvJEJ1c2NhQW5vTGlzdGFnZW1FbXByZXNhMSRncmRFbXByZXNhDxQrAAJkFwBkBT5jdGwwMCRjb250ZW50UGxhY2VIb2xkZXJDb250ZXVkbyRCdXNjYVNldG9yRW1wcmVzYTEkZ3JkRW1wcmVzYQ8UKwACZBcAZAU9Y3RsMDAkY29udGVudFBsYWNlSG9sZGVyQ29udGV1ZG8kQnVzY2FOb21lRW1wcmVzYTEkZ3JkRW1wcmVzYQ8UKwACZBcAZGIeDJ3CvUlArW3vV2fOJkmBjNaU&__EVENTVALIDATION=%2FwEWKQLsuvqUAgKrqYeWCwL85ODVCQLh%2B8LADwKyt5yADgKXzv7qAwLoidiqAgLNoLqVCALuyLaBBALT35jsCQKkm%2FKrCAKJstSWDgLa7a3WDAK%2FhJDBAgKQwOmAAQL11svrBgL2pYL%2FBwLbvOTpDQKs%2BL2pDAKRj6CUAgLiyvlTAsfh274GApidtf4EAv2zl%2BkKAp7ck9UGAoPz9b8MAtSuz%2F8KApb%2Fx9cCAvuVqsIIAszRg4IHArHo5ewMAoKkv6wLAue6oZcBArj2%2BtYPAp2N3cEFAr612a0BAqPMu5gHAqqynsAFAvrzvPwMAr%2FS5NkCArSz5PIOHWLjP%2FD3GHMwkLwI%2Bt6LJtKwm6A%3D&ctl00_contentPlaceHolderConteudo_AjaxPanelBuscaPostDataValue=ctl00_contentPlaceHolderConteudo_AjaxPanelBusca%2CActiveElement%2Cctl00_contentPlaceHolderConteudo_tabMenuEmpresaListada_tabSegmento%3B&ctl00$contentPlaceHolderConteudo$tabMenuEmpresaListada=%7B%22State%22%3A%7B%22SelectedIndex%22%3A2%7D%2C%22TabState%22%3A%7B%22ctl00_contentPlaceHolderConteudo_tabMenuEmpresaListada_tabNome%22%3A%7B%22Selected%22%3Afalse%7D%2C%22ctl00_contentPlaceHolderConteudo_tabMenuEmpresaListada_tabSegmento%22%3A%7B%22Selected%22%3Atrue%7D%7D%7D&=&ctl00$contentPlaceHolderConteudo$BuscaNomeEmpresa1$txtNomeEmpresa$txtNomeEmpresa=&ctl00$contentPlaceHolderConteudo$mpgPaginas_Selected=2&RadAJAXControlID=ctl00_contentPlaceHolderConteudo_AjaxPanelBusca&httprequest=true')

  select = doc.xpath('//select[@name="ctl00$contentPlaceHolderConteudo$BuscaSegmentoEmpresa1$cmbSegmento"]').first

  select.xpath('./option').each do |opt|
    next if opt['value'] == 'Selecione'
    puts("%-8s %s" % [opt['value'], opt.text])
  end
end

if __FILE__ == $0
  if ARGV[0]
    fetch_symbols_from_segment(ARGV[0])
  else
    fetch_segments
  end
end