mongodb-pre-split

gistfile1.txt

        #3. Pre-Splitting
        # export_size > 64K(records) and no data in dest collection
        chunksize = self.args.chunksize # default 64K items (for 64MBytes chunksize)
        maxshardkey = self.args.maxshardkey 

        if not(self.pre_src_count > chunksize and self.pre_to_count == 0):
            logging.info('we will not pre-split !')
            return 

        logging.info('we will pre-split by chunksize: %d, maxshardkey is: %d' % (chunksize, maxshardkey))
        splits = self.pre_src_count/chunksize
        shardkey_chunk_size = maxshardkey/splits
        #split
        for i in range(1, splits):
            split = i * shardkey_chunk_size
            toshard = random_shard()
            try:
                logging.info('split chunk %s @ %s' % (ns, str({shardkey:split}))  )
                self.dest_conn.admin.command('split', ns, middle={shardkey:split}) 
            except Exception as e:
                logging.warn('exception: ' + str(e))
        #move
        for i in range(1, splits):
            split = i * shardkey_chunk_size
            toshard = random_shard()
            try:
                logging.info('move chunk %s [find:%s] [to:%s]' % (ns, str({shardkey:split+1}), toshard ))
                self.dest_conn.admin.command('moveChunk', ns, find={shardkey:split+1}, to=toshard) 
            except Exception as e:
                logging.warn('exception: ' + str(e))