downloadUrlAsMarkdown.ts

// npm install turndown cheerio
import TurndownService from "turndown"
import { load } from "cheerio"


async function downloadUrlAsMarkdown(url: string, selector?: string) {
    try {
        const res = await fetch(url)
        if (!res.ok) {
            throw new Error(`${url} ${res.status} ${res.statusText}`)
        }
        const mimeType = res.headers.get("content-type")?.split(";")[0].toLowerCase()
        if (!mimeType) {
            throw new Error(`${url} unknown mime-type`)
        }
        if (!["text/html", "text/plain", "text/markdown"].includes(mimeType)) {
            throw new Error(`${url} not text or HTML: '${mimeType}'`)
        }
        let body = await res.text()
        let md = body
        // convert html to markdown if needed
        if (mimeType === "text/html") {
            if (selector) {
                const $ = load(body)
                const element = $(selector).html()
                if (!element) {
                    throw new Error(`${url} selector ${selector} doesn't get any content`)
                }
                body = element
            }
            // NOTE: parsing Vercel AI SDK and Mastra's docs
            // doesn't actually work super well (titles are lost)... 
            // but it's good enough for our RAG
            const turndownService = new TurndownService()
            turndownService.remove(["script", "nav"])
            md = turndownService.turndown(body)
        }
        return md
    } catch (err) {
        console.error(err)
        throw (err)
    }
}