readline_gql_insert.ts

import * as fs from 'fs'
import * as readline from 'readline'

...

const readStream = fs.createReadStream('title.basics.tsv')

const rl = readline.createInterface({ input: readStream })
const client = new ApolloClient({
  link: httpLink,
  cache: new InMemoryCache()
})
let pool = []
const insert = (objs) => {
  return client.mutate({
    mutation: gql`
    mutation ($objs: [imdb_title_basic_insert_input!]!) {
      insert_imdb_title_basic(objects: $objs) {
        affected_rows
      }
    }`,
    variables: { objs }
  })
}
let lineCount = 0
let rows = []

rl.on('line', (line) => {
  if (lineCount++ <= 1400000) return
  const arr = line.split('\t').map(col => col == '\\N' ? null : col)
  const [tconst, titleType, primaryTitle, originalTitle, isAdult, startYear, endYear, runtimeMinutes, genres] = arr
  rows.push({
    tconst,
    titleType,
    primaryTitle,
    originalTitle,
    isAdult: Boolean(parseInt(isAdult)),
    startYear,
    endYear,
    runtimeMinutes,
    genres
  })
  if (lineCount % 1000 === 0) {
    pool.push(insert(rows).catch(() => { }))
    if (pool.length >= 50) {
      rl.pause()
      Promise.all(pool).then(() => {
        pool = []
        rl.resume()
      })

    }
    rows = null
    rows = []
  }
})

rl.on('pause',()=>console.log(lineCount))
rl.on('close', () => insert(rows))