Skip to content

Instantly share code, notes, and snippets.

View mrorii's full-sized avatar

Naoki Orii mrorii

View GitHub Profile
#!/usr/bin/perl
use strict;
use warnings;
use Web::Scraper;
use URI;
use Data::Dumper;
use File::Spec;
use utf8;
use open IO => ":utf8";
@mrorii
mrorii / hoge.pl
Created June 13, 2011 11:17
get raw unicode character's hex point
#!/usr/bin/perl
use strict;
use warnings;
use Encode;
use utf8;
# Usage: perl hoge.pl < INPUT_FILE
# Example: perl hoge.pl < ISCAS-S-C-3.txt
@mrorii
mrorii / gist:966045
Created May 11, 2011 07:28
updated code sample for detecting type of character (kanji, hiragana, katakana, ...)
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace CharsetClassification
{
public class Program
{
public static void PrintCharTypes(string str)
@mrorii
mrorii / gist:961963
Created May 9, 2011 02:47
calculate kl divergence between two documents
# refer to http://staff.science.uva.nl/~tsagias/?p=185
import re, math, collections
def tokenize(_str):
stopwords = ['and', 'for', 'if', 'the', 'then', 'be', 'is', 'are', 'will', 'in', 'it', 'to', 'that']
tokens = collections.defaultdict(lambda: 0.)
for m in re.finditer(r"(\w+)", _str, re.UNICODE):
m = m.group(1).lower()
if len(m) < 2: continue
@mrorii
mrorii / convertSubmissionTrack1.py
Created March 26, 2011 04:05
Conversion Tool Python (Track 1)
'''
Use this script for converting a textual Track1 prediction file into a
well formatted binary submission file.
The script accepts two arguments:
<prediction file> (input) and <submission file> (output)
The input prediction file should contain 6005940 lines, corresponding
to the 6005940 user-item pairs in the test set.
Each line contains a predicted score (a real number between 0 and 100).
@mrorii
mrorii / convertSubmissionTrack1.cpp
Created March 26, 2011 04:04
Conversion Tool C++ (Track 1)
//Use this program for converting a textual Track1 prediction file into a
//well formatted binary submission file.
//
//The program accepts two arguments:
//<prediction file> (input) and <submission file> (output)
//
//The input prediction file should contain 6005940 lines, corresponding
//to the 6005940 user-item pairs in the test set.
//Each line contains a predicted score (a real number between 0 and 100).
//The generated output file can be submitted to the KDD-Cup'11 evaluation
@mrorii
mrorii / gist:863621
Created March 10, 2011 05:34
get title from url
#!/usr/bin/perl
use strict;
use warnings;
use LWP;
my $user_agent = LWP::UserAgent->new();
my $page_url = "http://www.yahoo.co.jp";
my $res = $user_agent->get($page_url);
my $title;
#!/usr/bin/perl
use strict;
use warnings;
use Data::Dumper;
use utf8;
use Encode;
my $string = "株式会社DeNAは、株式会社グリーのほげほげfoobar
hoafooasdfkj株式会社Googleはほげあlksjd";
use strict;
use warnings;
print "hoge";
<?php
$base_directory = dirname(__FILE__) . '/..';
require_once($base_directory . '/libs/Testing/Selenium.php');
require_once($base_directory . '/libs/ConfigLoader.php');
class HtmlTest extends PHPUnit_Framework_TestCase {
public function __construct() {
parent::__construct();
global $base_directory;