Scraping YouTube search term completions using Node.js for marketing research

youtube-search-scraping.js

import { writeFile } from 'node:fs/promises'
import { dump } from 'js-yaml'

const getSuggestions = async function  (query) {
// https://stackoverflow.com/questions/11275365/youtube-api-search-auto-complete
  const endpoint = 'https://clients1.google.com/complete/search?client=youtube&gs_ri=youtube&ds=yt'
  const url = new URL(endpoint)
  url.searchParams.set('q', query)
  const response = await fetch(url)
  if (response.ok) {
    // https://developer.mozilla.org/en-US/docs/Web/API/Encoding_API/Encodings
    const buf = await response.arrayBuffer()
    const text = new TextDecoder('windows-1252').decode(buf)
    const [_, suggestions] = JSON.parse(text.substring(text.indexOf("["), text.indexOf("])") + 1))
    return suggestions.map(s => s[0])
  } else {
    throw new Error('Requesting suggestions failed for query ' + query)
  }
}

const alphabet = 'abcdefghijklmnopqrstuvwxyzöüä'.split('')
const scrapeTerms = async function (terms) {
  return await Promise.all(terms.map(async term => {
    const termSearch = getSuggestions(term)
    const alphabetSearch = alphabet.map(letter => getSuggestions(term + ' ' + letter))
    const suggestions = await Promise.all([termSearch, ...alphabetSearch])
    const uniqueSuggestions = [...new Set(suggestions.flat())].toSorted()
    return { term, results: uniqueSuggestions }
  }))
}

const terms = ['barrierefreiheit', 'a11y', 'accessibility', 'screenreader', 'barrierefreiheitsstärkungsgesetz', 'bfsg']
const results = await scrapeTerms(terms)
const str = dump(results)
await writeFile('results.yml', str)