Skip to content

Instantly share code, notes, and snippets.

from concurrent.futures import ProcessPoolExecutor
from requests_futures.sessions import FuturesSession
import json
import os
import pandas
from pandas import DataFrame
import requests
import time
from bs4 import BeautifulSoup
from collections import defaultdict, Counter
@N0taN3rd
N0taN3rd / chromeCrawling.js
Created May 5, 2017 16:36
use chrome to crawl
const CDP = require('chrome-remote-interface')
const Promise = require('bluebird')
const util = require('util')
const path = require('path')
const fs = require('fs-extra')
const filenameify = require('filenamify-url')
const JsDetector = require('./lib/jsDetector')
const detectLib = require('./lib/detectlibs')
const {apps, categories} = fs.readJsonSync('./lib/apps.json')
@N0taN3rd
N0taN3rd / html_rewriter.py
Created April 22, 2017 03:32
pywb proxy mods
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import sys
from six.moves.html_parser import HTMLParser
from six.moves.urllib.parse import urljoin, urlsplit, urlunsplit
body {
margin: 0;
text-align: center;
min-width:940px; /* changed; value was 660px */
font-size: small;
}
#header-wrapper {
width:940px; /* changed; value was 660px */
margin:0 auto 10px;
@N0taN3rd
N0taN3rd / detect.js
Created March 17, 2017 16:44
detecting window.document.domain
const cheerio = require('cheerio')
const fs = require('fs-extra')
const Promise = require('bluebird')
const path = require('path')
const generate = require('babel-generator')
const traverse = require('babel-traverse')
const {parse} = require('babylon')
const util = require('util')
const bTypes = require('babel-types')
const inlineCss = require('inline-css')
@N0taN3rd
N0taN3rd / urls.txt
Last active March 3, 2017 20:27
test
www.mendeley.com/sign-in/?routeTo=https%3A%2F%2Fwww.mendeley.com%2Fprofiles%2Fhelen-palmer
This file has been truncated, but you can view the full file.
!function (t) {
function e (i) {
if (n[i]) return n[i].exports;
var r = n[i] = {
exports: {},
id: i,
loaded: !1
};
return t[i].call(r.exports, r, r.exports, e), r.loaded = !0, r.exports
}
@N0taN3rd
N0taN3rd / archiveIt.index.js
Last active March 2, 2017 23:40
The case of helen-palmer
function (t, e, n) {
"use strict";
function i () {
this.attributes({
nextLink: void 0,
initialPath: WB_wombat_self_location.pathname.replace("network/", "") +
"co-authors/"
}), this.getNextCoAuthors = r, this.after("initialize", function () {
this.on(document, "ui:co-authors:view-more", r), this.getNextCoAuthors()
<div id="scriptParent">
<!--Test 2a, Script, local --><img src="pixel.png" title="test2a">
<!--Test 2b, Script, remote --><img src="pixel.png" title="test2b">
<script type="text/javascript">
//Test 2c, Script inline, DOM Manipution
document.addEventListener('DOMContentLoaded', function () {
var inlineScriptImage = new Image();
inlineScriptImage.src = "pixel.png";
inlineScriptImage.title = "test2c";
@N0taN3rd
N0taN3rd / cnn_js.unmangled.20160902230242.js
Created January 17, 2017 05:20
Lets Make The CNN Archive Great Again: Unmangled Mins
/*
const <name> = min file unmangled
properties
'Global CNN varible w/ property chain': [
{
prop: 'lhs' // CNN w/ property chain assignee
to: 'rhs' // Assigned to
}
]
*/