i.coffee

fs = require 'fs'
_ = require 'lodash'
hl = require 'highland'
async = require 'async'
readline = require 'readline'
BSON = require("bson").BSONPure.BSON

tz = require 'timezone'

zlib = require 'zlib'
deflate = (x)->
  [x[0],(zlib.deflateSync x[1]).toString('base64')]

dateTransform = (x)->
  x.created_time = tz(x.created_time, 'Asia/Tokyo')
  x.reuploaded_time = tz(x.reuploaded_time, 'Asia/Tokyo')
  x
removeFields = (x)->
  x._v = 1
  x = _.omit x,[
    'image_urls',
    'stats',
    'publicity',
    'age_limit',
    'is_manga',
    'is_liked',
    'favorite_id',
    'book_style',
    'metadata',
    'content_type'
    ]
  x.user = _.omit x.user,[
    'is_following'
    'is_follower'
    'is_friend'
    'profile_image_urls'
  ]
  [x.id,x]

serialize = (x)->[x[0],BSON.serialize(x[1],true,true)]


instream = fs.createReadStream('data.txt')

rl = readline.createInterface input: instream,terminal: false

out = {}

(hl 'line',rl)
.map(JSON.parse)
.map(dateTransform)
.map(removeFields)
.map(serialize)
.map(deflate)
.map (arr)->
  out[arr[0]] = arr[1]