Roman Mirr romiras

How to use

Assumed you have files with extension .ebk stored by Kies.

Install Ruby 2.3 or later

	require 'concurrent'
	require 'benchmark'

	max = 20 # number of tasks to process
	n_workers = 4 # number of concurrent workers
	results = [] #
	lambda = -> (iter, i) { results << ('%04d' % i); d = rand(0.005)+0.001; puts("Iteration #{iter}. sleep %.3f" % d); sleep(d) }
	workers = Array.new(n_workers, lambda)
	iter = 0

	# Reads .jsonl file and transforms it into file ready for bulk import to ES.

	require 'json'

	File.open(ARGV[1], 'wb') {\|f\|
	ARGF.each_line { \|line\|
	a = JSON.parse(line)
	out = {
	index: {
	"_id": a["_id"],

	# Concurrently classify items by type and batch them with multiple workers

	require 'logger'

	NUM_WORKERS = 10
	NUM_ITEMS = 12
	NUM_ITERATIONS = 5000

	Item = Struct.new(:id, :type)

	require 'concurrent'
	require 'benchmark'
	require 'logger'

	module Logging # Credits: https://stackoverflow.com/a/6768164/10118318
	# This is the magical bit that gets mixed into your classes
	def logger
	Logging.logger
	end

	split --verbose -l1000 dataset-bulk-30k.ndjson bulk.

	for f in bulk.??; do echo $f; curl -i -X POST localhost:9200/_bulk -H "Content-Type: application/x-ndjson" --data-binary @$f; done

	require_relative './logging'
	require_relative './async_consumer'

	def drain(data_store, queue_name, output_filename)
	json_store = File.open(output_filename, 'a')

	executor = lambda do \|class_name, args\|
	Logging.logger.info "class_name: #{class_name}, args: #{args}"

	json_store.write("#{ { class: class_name, args: args }.to_json }\n")

	# using ripgrep on macos
	for f in $(rg --files-with-matches -F '2.7.6/'); do sed -i '' 's/\/2\.7\.6\//\/2\.7\.7\//' $f; done