Grab dod root domains inscope of DOD, written by dee-see

grabdomains.rb

require 'open3'
require 'net/http'
require 'nokogiri'
require 'parallel'

# Requires the nokogiri gem (`gem install nokogiri`), the parallel gem (`gem install parallel`) and the psql client for PostgreSQL.

def crtsh
  Open3.pipeline_rw 'psql -t -h crt.sh -p 5432 -U guest certwatch', "sed -e 's:^ *::g' -e 's:^*\\.::g' -e '/^$/d'", 'sort -u', "sed -e 's:*.::g'" do |i, o, t|
    i.write("SELECT ci.NAME_VALUE NAME_VALUE FROM certificate_identity ci WHERE ci.NAME_TYPE = 'dNSName' AND reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower('%.mil'));")
    i.close
    t.join
    return o.read.lines.map(&:chomp)
  end
end

def defensedotgov
  pages = *('A'..'Z')
  pages << '0-9'
  Parallel.map(pages, in_threads: 10) do |page|
    body = Net::HTTP.get(URI("https://www.defense.gov/Resources/Military-Departments/A-Z-List/?page=#{page}"))
    Nokogiri::HTML(body).css('.DGOVListLink, .DGOVListLinkAlt').children.map { |el| el.attributes['href'].value.gsub(%r{^https?://}, '') }
  end.flatten
end

puts crtsh.concat(defensedotgov).sort.uniq