Working with CJK encodings

test_cjk.py

from array import array
from jarray import zeros
from java.nio import ByteBuffer, CharBuffer
from java.nio.charset import Charset, CodingErrorAction  # FIXME actually use CodingErrorAction

shift_jis = Charset.forName("Shift_JIS")
decoder = shift_jis.newDecoder()
jis_data = open("shift_jis.txt", "rb").read()         # data encoded in shift_jis format
utf_8_data = open("shift_jis-utf8.txt", "rb").read()  # same data, encoded as utf-8
input_buffer = ByteBuffer.wrap(array("b", jis_data))  # FIXME break into chunks
output_buffer = CharBuffer.wrap(zeros(len(jis_data) * 2, "c")) # FIXME support overflows, use decoder.averageCharsPerByte())
decoder.decode(input_buffer, output_buffer, True)
pos = output_buffer.position()
output_buffer.rewind()
decoded = output_buffer.subSequence(0, pos).toString()  # FIXME build up with a StringBuilder

print type(decoded), decoded
print type(utf_8_data), utf_8_data
assert decoded.encode("utf-8") == utf_8_data