scrape-deviantart.js

import fs from 'fs'
import path from 'path'
import qs from 'querystring'
import httpsProxyAgent from 'https-proxy-agent'
import axios from 'axios'

const outDir = './images/'
const baseUrl = 'https://www.deviantart.com/_napi/da-browse/api/tags'
const userAgent = 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11'
let proxies = [
  'http://95.216.240.148:808',
  'http://95.216.156.195:443',
  'http://5.1.53.46:8080',
  'http://144.217.74.219:3128',
  'http://178.62.232.215:8080',
  'http://167.114.247.229:9999',
  'http://136.243.47.220:3128',
  'http://51.158.106.54:8811',
  'http://51.158.123.35:8811',
  'http://163.172.189.32:8811',
  'http://51.158.68.133:8811',
  'http://51.158.111.229:8811'
]

const tags = [
  'games',
  'gaming',
  'videogame',
  'leagueoflegends',
  'paintings',
  'digitalart'
]

function pageUrl(tag, offset) {
  const query = qs.stringify({
    init: 'false',
    order: 'popular-all-time',
    limit: '120',
    tag,
    offset
  })

  return `${baseUrl}?${query}`
}

function createProxyAgent() {
  const proxy = proxies[Math.floor(Math.random() * proxies.length)]
  console.log('using proxy url', proxy)
  return httpsProxyAgent(proxy)
}

async function apiRequest(url) {
  console.log(url)
  const response = await axios(url, {
    headers: {
      'User-Agent': userAgent
    },
    json: true,
    httpsAgent: createProxyAgent(),
    timeout: 5000
  })

  return response.data
}

async function downloadImage(id, url, ext) {
  console.log('downloading image', id, url)

  let response

  try {
    response = await axios(url, {
      headers: {
        'User-Agent': userAgent
      },
      responseType: 'stream'
    })
  } catch (err) {
    console.log('failed', id)
    return
  }

  const imgPath = path.resolve(outDir, `${id}${ext}`)

  if (fs.existsSync(imgPath)) {
    return
  }

  const stream = fs.createWriteStream(imgPath)
  response.data.pipe(stream)

  return new Promise((resolve, reject) => {
    stream.on('finish', resolve)
    stream.on('error', reject)
  }).then(() => {
    console.log('success', id)
  }).catch(err => {
    console.log('error', id)
    return
  })
}

function fetchImages(page) {
  return Promise.all(page.deviations
    .filter(deviation => deviation.isDownloadable)
    .filter(deviation => deviation.media && deviation.media.token && deviation.media.token.length !== 0)
    .map(deviation => {
      const {baseUri, prettyName, token, types} = deviation.media

      let url = baseUri
      if (types && types.length !== 0) {
        let maxWidth = 0
        let urlPart = ''

        for (const type of types) {
          if (!type.c) {
            continue
          }

          if (type.w > maxWidth) {
            maxWidth = type.w
            urlPart = type.c.replace('<prettyName>', prettyName)
          }
        }

        if (maxWidth !== 0) {
          url = `${url}/${urlPart}`
        }
      }

      url = `${url}?token=${token[0]}`
      const ext = path.extname(baseUri)
      return downloadImage(deviation.deviationId, url, ext)
    }))
}

async function run() {
  let offset = 0
  let tag = tags[0]

  while (true) {
    let page

    try {
      page = await apiRequest(pageUrl(tag, offset))
    } catch (err) {
      console.log('failed, retrying')
      continue
    }

    await fetchImages(page)
    offset = page.nextOffset
    if (!offset) {
      tag = tags[0]
      tags.shift()
      offset = 0
      console.log('using tag', tag)
    }

    console.log('current offset', offset)
  }
}

run()
.then(() => console.log('all done'))