ninoseki

require "json"
require 'certificate-transparency-client'
require "parallel"

ct = CT::Client.new "https://ct.example.org"
list = JSON.parse(File.read("all_logs_list.json"))
logs = list.dig("logs") || []
urls = logs.map { |log| "https://" + log.dig("url") }.map { |url| url[0..-2] }

Parallel.each(urls) do |url|

ninoseki

# frozen_string_literal: true

require "faye/websocket"
require "eventmachine"
require "json"

EM.run {
  ws = Faye::WebSocket::Client.new("ws://127.0.0.1:4000", nil, ping: 10)

  ws.on :message do |event|

ninoseki

16:05:50.128 [error] [Honeybadger] Mandatory config key :api_key not set
16:05:50.159 [info] Starting web server on port 4000...
16:05:50.171 [info] Starting Elixir.Certstream.ClientManager...
16:05:50.171 [info] Starting Elixir.Certstream.CertifcateBuffer...
16:05:50.172 [info] Initializing CT Watchers...
16:05:50.365 [info] Worker #PID<0.293.0> started with url ct.googleapis.com/logs/argon2017/.
16:05:50.365 [info] Worker #PID<0.294.0> started with url ct.googleapis.com/logs/argon2018/.
16:05:50.365 [info] Worker #PID<0.295.0> started with url ct.googleapis.com/logs/argon2019/.
16:05:50.366 [info] Worker #PID<0.296.0> started with url ct.googleapis.com/logs/argon2020/.
16:05:50.366 [info] Worker #PID<0.297.0> started with url ct.googleapis.com/logs/argon2021/.

ninoseki

16:05:47.233 [error] [Honeybadger] Mandatory config key :api_key not set
16:05:47.261 [info] Starting web server on port 4040...
16:05:47.274 [info] Starting Elixir.Certstream.ClientManager...
16:05:47.274 [info] Starting Elixir.Certstream.CertifcateBuffer...
16:05:47.275 [info] Initializing CT Watchers...
16:05:47.275 [info] Sending GET request to https://www.gstatic.com/ct/log_list/all_logs_list.json
16:05:47.564 [info] Worker #PID<0.293.0> started with url ct.googleapis.com/logs/argon2017/.
16:05:47.564 [info] Sending GET request to https://ct.googleapis.com/logs/argon2017/ct/v1/get-entries?start=0&end=1024
16:05:47.710 [info] Worker #PID<0.293.0> found batch size of 32.
16:05:47.710 [info] Worker #PID<0.298.0> started with url ct.googleapis.com/logs/argon2018/.

ninoseki

15:14:24.856 [error] [Honeybadger] Mandatory config key :api_key not set
15:14:24.882 [info] Starting web server on port 4000...
15:14:24.898 [info] Starting Elixir.Certstream.ClientManager...
15:14:24.898 [info] Starting Elixir.Certstream.CertifcateBuffer...
15:14:24.899 [info] Initializing CT Watchers...
15:14:25.080 [info] Worker #PID<0.283.0> started with url ct.googleapis.com/logs/argon2017/.
15:14:25.080 [info] Worker #PID<0.284.0> started with url ct.googleapis.com/logs/argon2018/.
15:14:25.080 [info] Worker #PID<0.285.0> started with url ct.googleapis.com/logs/argon2019/.
15:14:25.080 [info] Worker #PID<0.286.0> started with url ct.googleapis.com/logs/argon2020/.
15:14:25.080 [info] Worker #PID<0.287.0> started with url ct.googleapis.com/logs/argon2021/.

ninoseki

# most abused registrars (source: https://www.spamhaus.org/news/images/Q2-2019/2019-Q2-Spamhaus-Botnet-Threat-Update.pdf)
Namecheap
RegRU
NameSilo
PDR
Openprovider
GMO
RU-Center
WebNic.cc
Xin Net

ninoseki

# frozen_string_literal: true

require "http"

TLD_LIST = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"

def tlds
  res = HTTP.get(TLD_LIST)
  return [] if res.code != 200

ninoseki

==> default: Checking for guest additions in VM...
==> default: Mounting shared folders...
    default: /vagrant => /private/tmp/yeti
    default: /opt/yeti => /private/tmp/yeti
==> default: Detected mount owner ID within mount options. (uid: 1001 guestpath: /opt/yeti)
==> default: Detected mount group ID within mount options. (gid: 1001 guestpath: /opt/yeti)
==> default: Running provisioner: shell...
    default: Running: inline script
    default: useradd: UID 1001 is not unique
    default: id:

ninoseki

# frozen_string_literal: true

require "zip"

def check(path, keyword)
  Zip::File.open(path) do |zip_file|
    zip_file.each do |entry|
      next if entry.name.end_with? "/"
      next if File.extname(entry.name).empty?

ninoseki

# frozen_string_literal: true

require "down"

def domains
  File.readlines("domains.txt").map(&:chomp).reject(&:empty?)
end

def download_favicon(domain, download_to)
  url = "http://www.google.com/s2/favicons?domain=#{domain}"