A scraper to scrape all scrabble words from scrabblewordfinder.org written in Haskell.

scrabble_scraper.hs

#!/usr/bin/env cabal
{- cabal:
  build-depends: base, scalpel
-}
{-# LANGUAGE OverloadedStrings #-}

import Control.Monad
import Data.Char
import Data.List
import Data.Maybe
import System.IO
import Text.HTML.Scalpel

loadUrls :: IO [String]
loadUrls = do
  let url = "https://scrabblewordfinder.org/word-list"
  urls <- fromJust <$> scrapeURL url (attrs "href" "a")
  let urls1 = filter ((> 15) . length) urls
  let urls2 = fixUrl <$> urls1
  return urls2

fixUrl :: String -> String
fixUrl url
  | "/" `isPrefixOf` url = "https://scrabblewordfinder.org" ++ url
  | otherwise = url

loadWords :: String -> IO [String]
loadWords url = do
  words <- fromJust <$> scrapeURL url (texts ("a" @: [hasClass "wordWrapper"]))
  return $ fixWord <$> words

fixWord :: String -> String
fixWord = map toUpper . takeWhile isAlpha . dropWhile (not . isAlpha)

mergeWorLists :: [[String]] -> [String]
mergeWorLists = map head . group . sort . concat

main :: IO ()
main = do
  urls <- loadUrls
  wordLists <- forM urls $ \url -> do
    hPutStrLn stderr $ "Scrapping " ++ url ++ " ..."
    words <- loadWords url
    hPutStrLn stderr $ "Found " ++ show (length words) ++ " words."
    hPutStrLn stderr $ unwords $ take 10 words
    hPutStrLn stderr "\n\n\n"
    return words

  hPutStrLn stderr "Saving words ..."
  putStr $ unlines $ mergeWorLists wordLists

  hPutStrLn stderr "Done."