/r/destinythegame Sentiment Analysis

index.js

/* eslint-disable no-restricted-syntax */
/* eslint-disable unicorn/consistent-function-scoping */
/* eslint-disable no-await-in-loop */

const axios = require(`axios`);

const language = require(`@google-cloud/language`);
const client = new language.LanguageServiceClient();

const createCsvWriter = require(`csv-writer`).createObjectCsvWriter;
const csvWriter = createCsvWriter({
	path: `./data.csv`,
	header: [
		{ id: `id`, title: `ID` },
		{ id: `timestamp`, title: `TIMESTAMP` },
		{ id: `sentiment`, title: `SENTIMENT` },
		{ id: `title`, title: `TITLE` },
		{ id: `score`, title: `SCORE` },
		{ id: `awards`, title: `AWARDS` },
		{ id: `flair`, title: `FLAIR` },
		{ id: `deleted`, title: `DELETED` },
	],
});

// ========= USER SETTINGS =========;

const START_DATE = 1587168000; // Midnight UTC on April 18th, 2020
const SUB_AMOUNT = 21600; // 6 hours

const CUTOFF = 1354320000; // Midnight UTC on December 1, 2012

// ========= MAIN =========

async function main() {
	const getSentiments = async (before, after) => {
    
		// get posts within a time period from the subreddit
		const response = await axios({
			method: `get`,
			url: `https://api.pushshift.io/reddit/search/submission/`,
			params: {
				subreddit: `DestinyTheGame`,
				size: 500, // max size from API is 500
				before,
				after,
			},
		});

		// create a hodler array
		const sentiments = [];

		// for each post in the response
		for (const post of response.data.data) {
			// console.log(post);

			try {
				// get the title
				const text = post.title;
				const document = {
					content: text,
					type: `PLAIN_TEXT`,
				};

				// get the sentiment
				const [result] = await client.analyzeSentiment({
					document,
				});

				// create a payload
				const payload = {
					id: post.id,
					timestamp: post.created_utc,
					sentiment: result.documentSentiment.score,
					title: post.title,
					awards: post.total_awards_received,
					score: post.score,
					flair: post.link_flair_text,
					deleted: !!post.banned_by,
				};

				// sleep to not overload google

				await ((ms) => {
					return new Promise((resolve) => {
						setTimeout(resolve, ms);
					});
				})(75);

				// add it to the holder array
				console.log(payload.title);
				sentiments.push(payload);
			} catch {
				console.error(
					`PROBLEM WITH ${post.id} @ ${post.created_utc}`
				);
			}
		}

		console.log(`==========`, sentiments.length);

		// write the array to a CSV
		await csvWriter.writeRecords(sentiments);
	};

	// loop until the subreddit's start date
	let startDate = START_DATE;
	let endDate = startDate - SUB_AMOUNT;

	while (startDate > CUTOFF) {
		console.log(startDate, endDate);

		await getSentiments(startDate, endDate);
		startDate = endDate;
		endDate -= SUB_AMOUNT;
	}
}

main();

package.json

{
  "name": "destiny-sentiment",
  "version": "1.0.0",
  "main": "index.js",
  "license": "MIT",
  "scripts": {
    "start": "node index.js"
  },
  "dependencies": {
    "@google-cloud/language": "^4.0.0",
    "@j0hnm4r5/eslint-config": "^1.0.7",
    "axios": "^0.19.2",
    "csv-writer": "^1.6.0",
    "eslint": "^6.8.0"
  }
}