Skip to content

Instantly share code, notes, and snippets.

@nulltask
Created September 14, 2015 09:23
Show Gist options
  • Select an option

  • Save nulltask/3b8691e1abca9ede1d79 to your computer and use it in GitHub Desktop.

Select an option

Save nulltask/3b8691e1abca9ede1d79 to your computer and use it in GitHub Desktop.
var sqlite3 = require("sqlite3").verbose();
var express = require('express');
var db = new sqlite3.Database('name.db');
var app = express();
app.use(express.static('public'));
app.get('/all', function(req, res) {
res.send(require('./all.json'));
});
app.get('/chars/:char/prev', function(req, res) {
var query = [
'SELECT character, prev_character, COUNT(*) AS count',
'FROM character_connections',
'WHERE character = \'' + req.params.char + '\'',
'GROUP BY prev_character',
'ORDER BY count DESC'
].join(' ');
db.all(query, function(err, rows) {
res.send(rows);
})
});
app.get('/chars/:char/next', function(req, res) {
var query = [
'SELECT character, next_character, COUNT(*) AS count',
'FROM character_connections',
'WHERE character = \'' + req.params.char + '\'',
'GROUP BY next_character',
'ORDER BY count DESC'
].join(' ');
db.all(query, function(err, rows) {
res.send(rows);
})
});
app.listen(process.env.PORT || 3000, function() {
console.log('listening');
});
source 'https://rubygems.org'
gem 'sqlite3'
var cheerio = require('cheerio');
var request = require('superagent');
var Batch = require('batch');
var iconv = require('iconv-lite');
var urls = [
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=aa',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ka',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=sa',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ta',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=na',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ha',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ma',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ya',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ra',
'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=wa'
]
var batch = new Batch();
urls.forEach(function(url) {
batch.push(function(done) {
request
.get(url)
.end(onload);
function onload(err, res) {
console.log(res);
var text = iconv.decode(res.text, 'euc-jp').toString('utf-8');
var $ = cheerio.load(text);
var tr = $('td[width="25%"]').map(function() {
console.log($(this).text());
});
}
done(null, url);
});
});
batch.end(function(err, res) {
console.log(res);
});
var a = 'td[width="25%"]';
require 'json'
require 'sqlite3'
db = SQLite3::Database.new('name.db')
all = JSON.parse(open('./all.json').read)
rows = db.execute <<-SQL
CREATE TABLE character_connections (
id INTEGER PRIMARY KEY,
prev_character TEXT,
character TEXT,
next_character TEXT
);
SQL
db.transaction do
all.each do |name|
next if name.include?('(')
chars = name.chars
chars.each_with_index do |char, index|
prev_char = index == 0 ? nil : chars[index - 1]
next_char = chars[index + 1]
p "#{prev_char} -> #{char} -> #{next_char}"
db.execute('INSERT INTO character_connections (character, prev_character, next_character) VALUES (?, ?, ?)', char, prev_char, next_char)
end
end
end
{
"scripts": {
"start": "node app"
},
"dependencies": {
"batch": "^0.5.2",
"cheerio": "^0.19.0",
"express": "^4.13.3",
"iconv-lite": "^0.4.11",
"sqlite3": "^3.1.0",
"superagent": "^1.3.0"
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment