avnersorek

/**
The script is designed to be respectful of Reddit's API limits while comprehensively collecting both posts 
and comments for offline analysis or archival purposes.

I have used it to scrape subreddits and throw the text in NotebookLLM.
Takes like 30 minutes to download a subreddit (depending on how busy it is) since there are a lot of rate limits. 

1. Scrapes recent posts from a specified subreddit (last 30 days) using Reddit's JSON API
2. Fetches comments for each post along with the post content
3. Saves everything to a text file named {subreddit}_posts.txt with formatted content including:

avnersorek

'use strict';
const cp = require('child_process');
const showdown = require('showdown');
const config = require('my-config');

const requestTimeout = config.get('app:requestTimeoutMs');

const converter = new showdown.Converter();

// in the master process

avnersorek

'use strict';
const showdown = require('showdown');
const converter = new showdown.Converter();

function markdownToHtml(markdown) {
  return converter.makeHtml(markdown);
}

module.exports = {
  markdownToHtml

avnersorek

# Place this file in the .ebextensions folder of your project
files:
  "/home/ec2-user/logdna.sh" :
    mode: "000777"
    owner: root
    group: root
    content: |
      #!/bin/sh
      echo "[logdna]
      name=LogDNA packages

avnersorek

import {go, chan, take, put} from 'js-csp';
let channel = chan();

go(function* () {
  const text = yield take(ch);
  console.log('A > RECEIVED:', text);
});

go(function* () {
  const text = yield take(ch);

avnersorek

var redisGet = Q.nbind(redisClient.get, redisClient);
var mysqlGet = Q.nbind(mysqlClient.get, mysqlClient);

function GetUser(userId) {
  return redisGet(userId)
  .then(function(redisResult){
    if (redisResult !== null) return redisResult;
    else return mysqlGet(userId);
  });
}

avnersorek

function GetUser(userId, callback) {
  redisClient.get(userId, function(redisError, redisResult){
    if (redisError !== null) callback(redisError, null);
    else if (redisResult !== null) callback(null, redisResult);
    else {
      mysqlClient.getUser(userId, function(mysqlError, mysqlResult){
        if (mysqlError !== null) callback(mysqlError, null);
        else callback(null, mysqlResult);
      });
    }

avnersorek

public User Get(string userId) {
  User cachedUser = redisClient.GetUser(userId);
  if (cachedUser !== null) return cachedUser;
  else return mySqlClient.GetUser(userId);
}

avnersorek

var crypto = require('crypto');

var DOORBELL_SECRET = '--your secret--';
var DOORBELL_KEY = '--your app key--';

function getOptions() {
  var options = {
    appKey: DOORBELL_KEY,
    timestamp: Math.floor(new Date() / 1000),
    token: crypto.randomBytes(16).toString('hex')

avnersorek

"use strict";
var request = require('superagent');
var Q = require('q');

var config = {
  github : {
    // https://help.github.com/articles/creating-an-access-token-for-command-line-use/
    user : GITHUB_USERNAME,
    password : GITHUB_TOKEN
  },