nyaa_comments.js

const httpFetch = require('node-fetch')
const { JSDOM } = require('jsdom')
const express = require('express')
const morgan = require('morgan')
const NodeCache = require('node-cache')
const { Feed } = require('feed')

const {
  PORT = 8080
} = process.env

const app = express()

app.use(morgan('dev'))

/// caching

const cache = new NodeCache({
  stdTTL: 60 * 10,
  checkperiod: 60 * 3
})

/// limit network requests

let lastNetRsq = Promise.resolve()
function fetch(...args) {
  const lastrsq = lastNetRsq

  const promise = Promise.resolve()
    .then(async () => {
      await lastrsq
      console.log('now requesting')
      return await httpFetch(...args)
    })
  
  lastNetRsq = promise
  
  return promise
}

/// logic

function commentsState(document) {
  return Array.from(document.querySelectorAll('tr > td:nth-child(2)'))
    .map(tr => tr.children)
    .map(([acom, alink]) => ({
      title: (alink || acom).textContent,
      link: (alink || acom).href,
      comments: alink ? +acom.textContent : 0
    }))
    .map(o => [o.link, o])
    .reduce((o, [k, v]) => (o[k] = v, o), {}) // Object.fromEntries
}

function commentsDiff(lastState, newState) {
  return Object.entries(newState)
    .map(([link, { comments, ...fields }]) =>
      ({ comments: comments - ((lastState[link] || {}).comments || 0), ...fields })
    )
    .filter(({ comments }) => comments > 0)
}

function comments(document) {
  return Array.from(document.querySelectorAll('#collapse-comments > div > div'))
    .map(div => ({
      user: div.querySelector('a').textContent,
      avatar: div.querySelector('img.avatar').src,
      date: new Date(div.querySelector('div.comment-details > a > small').dataset['timestamp'] * 1000),
      content: div.querySelector('div.comment-content').innerHTML,
    }))
}

/// network out

async function getStateFor(user) {
  const link = `https://nyaa.si/user/${user}?s=comments&o=desc`

  console.log(`Requesting comments state for ${user}`)
  const res = await fetch(link)
  if (res.status === 404) return { status: 'not-found' }
  if (!res.ok) return { status: 'error' }

  const text = await res.text()

  const dom = new JSDOM(text)
  const document = dom.window.document
  
  try {
    return { status: 'ok', state: commentsState(document) }
  } catch (e) {
    console.error(`Error for user ${user}`, e)
    return { status: 'error' }
  }
}

async function getComments(partialUrl) {
  const link = `https://nyaa.si${partialUrl}`

  console.log(`Requesting comments for ${partialUrl}`)
  const res = await fetch(link)
  if (res.status === 404) return { status: 'not-found' }
  if (!res.ok) return { status: 'error' }

  const text = await res.text()

  const dom = new JSDOM(text)
  const document = dom.window.document

  try {
    return { status: 'ok', comments: comments(document) }
  } catch (e) {
    console.error(`Error for url ${partialUrl}`, e)
    return { status: 'error' }
  }
}

/// network in

function logsRss(user, logs) {
  const feed = new Feed({
    title: `Nyaa.si ${user} comment feed`,
    link: `https://nyaa.si/user/${user}`,
    description: `Comment feed for nyaa.si user ${user}`
  })

  logs.forEach(({ title, link, comments, avatar, content, date, user }) =>
    feed.addItem({
      title: `${title} - User ${user} commented`,
      image: avatar,
      date,
      link: `https://nyaa.si${link}`,
      content
    })
  )

  return feed.rss2()
}

app.get('/user/:user', async (req, res) => {
  const { user } = req.params

  let {
    status: cstatus,
    state: cached = {},
    logs = []
  } = await cache.get(user) || {}

  if (cstatus === 'not-found')
    return res.sendStatus(404)

  if (cstatus === 'ok') {
    res.set('Content-Type', 'application/rss+xml');
    return res.send(logsRss(user, logs))
  }

  const { status, state } = await getStateFor(user)

  if (status === 'error')
    return res.sendStatus(500)
  if (status === 'not-found') {
    cache.set(user, { status: 'not-found' }, 60 * 60) //1h
    return res.sendStatus(404)
  }
  
  const diff = commentsDiff(cached, state)

  const update = (await Promise.all(
    diff.map(obj => getComments(obj.link).then(res => [obj, res]))
  ))
    .filter(([, { status }]) => status === 'ok')
    .map(([obj, { comments }]) => ({ ...obj, comments }))

  const newLogs =
    update.map(({ comments, ...torrFields }) => comments.map(comm => ({ ...torrFields, ...comm })))
      .reduce((acc, cur) => [...acc, ...cur], []) // Array.prototype.flat()
      .sort(({ date: d1 }, { date: d2 }) => d2 - d1)

  logs = [...newLogs, ...logs]

  cache.set(user, { status: 'ok', state, logs })

  res.set('Content-Type', 'application/rss+xml');
  res.send(logsRss(user, logs))
})

app.listen(PORT, () => console.log(`App running on port ${PORT}!`))