full_reload.js

const cheerio = require('cheerio')
const request = require('request')
const mongoose = require('mongoose')
const config = require('config')
const Publication = require('./server/models/publication')

mongoose.Promise = global.Promise

let archives = [
  ["2016", "12"],
  ["2017", "01"],
  ["2017", "02"]
]

const getStories = (year, month) => {
  return new Promise((resolve, reject) => {
    request.get(`https://medium.com/qlik-branch/archive/${year}/${month}`, (error, response, body) => {
      try {
        let $ = cheerio.load(body)
        let urls = $('h3 a').map((i, a) => $(a).attr('href')).get()
        Promise.all(urls.map(url => getStory(url)))
          .then(resolve)
          .catch(reject)
      } catch (e) {
        reject(e)
      }
    })
  })
}

const getStory = (url) => {
  return new Promise((resolve, reject) => {
    request.get(url, (error, response, body) => {
      try {
        if (error) {
          console.error(error)
          reject(e)
        } else {
          let $ = cheerio.load(body)
          let publication = new Publication()
          let content = $('div.section-inner')
          content.find('h1').remove()
          publication.title = $('meta[name="title"]').attr('content')
          //publication.title = publication.title.encode('ascii','ignore')
          publication.author = $('meta[name="author"]').attr('content')
          publication.link = $('link[rel="canonical"]').attr('href')
          publication.tags = $('ul.tags a').map((i, a) => $(a).text()).get().join(', ')
          publication.published = new Date($('meta[property="article:published_time"]').attr('content'))
          publication.published_num = publication.published.getTime()
          publication.content = content.html()
          publication.mediumId = publication.link.substring(publication.link.lastIndexOf('-') + 1)
          publication.plaintext = content.text()
          publication.short_description = $('meta[name="description"]').attr('content')
          publication.image = $('meta[property="og:image"]').attr('content') || "/attachments/default/publication.png"
          publication.approved = true
          publication.save()
            .then(resolve)
            .catch(reject)
        }
      } catch (e) {
        console.error(e.message)
        reject(e)
      }
    })
  })
}

mongoose.connect(config.mongoconnectionstring)
  .then(connection => {
    Promise.all(archives.map(archive => getStories(archive[0], archive[1])))
      .then(() => {
        console.log('done')
        mongoose.connection.close()
      })
      .catch(error => {
        console.error(error)
        mongoose.connection.close()
      })
  })
  .catch(error => {
    console.error(error)
  })

publication.js

var mongoose = require('mongoose');
var Schema = mongoose.Schema;

var publicationSchema = new Schema({
  title: String,
  short_description: String,
  mediumId: String,
  content: String,
  plaintext: String,
  link: String,
  image: String,
  tags: String,
  published: Date,
  published_num: Number,
  author: String,
  checksum: String,
  approved: Boolean
});

module.exports = mongoose.model('publication', publicationSchema)