Created
September 14, 2015 09:23
-
-
Save nulltask/3b8691e1abca9ede1d79 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var sqlite3 = require("sqlite3").verbose(); | |
| var express = require('express'); | |
| var db = new sqlite3.Database('name.db'); | |
| var app = express(); | |
| app.use(express.static('public')); | |
| app.get('/all', function(req, res) { | |
| res.send(require('./all.json')); | |
| }); | |
| app.get('/chars/:char/prev', function(req, res) { | |
| var query = [ | |
| 'SELECT character, prev_character, COUNT(*) AS count', | |
| 'FROM character_connections', | |
| 'WHERE character = \'' + req.params.char + '\'', | |
| 'GROUP BY prev_character', | |
| 'ORDER BY count DESC' | |
| ].join(' '); | |
| db.all(query, function(err, rows) { | |
| res.send(rows); | |
| }) | |
| }); | |
| app.get('/chars/:char/next', function(req, res) { | |
| var query = [ | |
| 'SELECT character, next_character, COUNT(*) AS count', | |
| 'FROM character_connections', | |
| 'WHERE character = \'' + req.params.char + '\'', | |
| 'GROUP BY next_character', | |
| 'ORDER BY count DESC' | |
| ].join(' '); | |
| db.all(query, function(err, rows) { | |
| res.send(rows); | |
| }) | |
| }); | |
| app.listen(process.env.PORT || 3000, function() { | |
| console.log('listening'); | |
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| source 'https://rubygems.org' | |
| gem 'sqlite3' |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var cheerio = require('cheerio'); | |
| var request = require('superagent'); | |
| var Batch = require('batch'); | |
| var iconv = require('iconv-lite'); | |
| var urls = [ | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=aa', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ka', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=sa', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ta', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=na', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ha', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ma', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ya', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=ra', | |
| 'http://www.pachi.ac/~multi/cgi-bin/familyname/display.cgi?mode=wa' | |
| ] | |
| var batch = new Batch(); | |
| urls.forEach(function(url) { | |
| batch.push(function(done) { | |
| request | |
| .get(url) | |
| .end(onload); | |
| function onload(err, res) { | |
| console.log(res); | |
| var text = iconv.decode(res.text, 'euc-jp').toString('utf-8'); | |
| var $ = cheerio.load(text); | |
| var tr = $('td[width="25%"]').map(function() { | |
| console.log($(this).text()); | |
| }); | |
| } | |
| done(null, url); | |
| }); | |
| }); | |
| batch.end(function(err, res) { | |
| console.log(res); | |
| }); | |
| var a = 'td[width="25%"]'; |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| require 'json' | |
| require 'sqlite3' | |
| db = SQLite3::Database.new('name.db') | |
| all = JSON.parse(open('./all.json').read) | |
| rows = db.execute <<-SQL | |
| CREATE TABLE character_connections ( | |
| id INTEGER PRIMARY KEY, | |
| prev_character TEXT, | |
| character TEXT, | |
| next_character TEXT | |
| ); | |
| SQL | |
| db.transaction do | |
| all.each do |name| | |
| next if name.include?('(') | |
| chars = name.chars | |
| chars.each_with_index do |char, index| | |
| prev_char = index == 0 ? nil : chars[index - 1] | |
| next_char = chars[index + 1] | |
| p "#{prev_char} -> #{char} -> #{next_char}" | |
| db.execute('INSERT INTO character_connections (character, prev_character, next_character) VALUES (?, ?, ?)', char, prev_char, next_char) | |
| end | |
| end | |
| end |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "scripts": { | |
| "start": "node app" | |
| }, | |
| "dependencies": { | |
| "batch": "^0.5.2", | |
| "cheerio": "^0.19.0", | |
| "express": "^4.13.3", | |
| "iconv-lite": "^0.4.11", | |
| "sqlite3": "^3.1.0", | |
| "superagent": "^1.3.0" | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment