maxjustus

// npm install @clickhouse/client parquetjs
const { createClient } = require('@clickhouse/client');
const { ParquetSchema, ParquetTransformer } = require('parquetjs');
const { Readable, Transform } = require('stream');

// Transform stream to convert from object mode to binary
class BinaryModeTransform extends Transform {
  constructor() {
    super({
      objectMode: false // Output in binary mode for ClickHouse js client

maxjustus

-- uses pk assuming pk is y, z
select * from x where (y, z) in (select y, z from f);
-- does not use pk
select * from x where (y, z) in (select (y, z) from f);

maxjustus

-- https://fiddle.clickhouse.com/a43809c0-cd97-47e7-ab0b-ca85812a7590 for running example
drop table if exists x;
create temporary table x (id Int64, key String, value String, time Int64);

-- id 1
insert into x values (1, 'k-a', 'val one', 1), (1, 'k-a', 'val two', 2);
insert into x values (1, 'k-b', 'val two', 2), (1, 'k-b', 'val one', 3);
insert into x values(1, 'k-c', 'a', 2), (1, 'k-c', 'b', 30), (1, 'k-c', 'c', 1);

-- id 2

maxjustus

-- https://fiddle.clickhouse.com/43192eb8-cf28-4654-ad07-b7a78bdcb872
CREATE TABLE users (uid Int16, names Array(Tuple(name String, age Int16))) 
  ENGINE=EmbeddedRocksDB
    primary key uid;

INSERT INTO users VALUES (1231, [('John', 33)]);
INSERT INTO users VALUES (6666, [('Ksenia', 48)]);
INSERT INTO users VALUES (8888, [('Alice', 50)]);

SELECT * FROM users;

maxjustus

# The clickhouse client --format flag specifies the output format of query results that
# get printed to the console by clickhouse client. There are a many options. See: https://clickhouse.com/docs/en/interfaces/formats
# The Null format will not print any query results for the console, which in this case is useful because we are only
# interested in analyzing the trace logs that are printed to the console and any query output just adds noise.

# inline query example:
clickhouse client --send-logs-level="trace" --database=test --progress --query="select* from system.numbers limit 10;" --format=Null

# Multi statement query file example:
echo "

maxjustus

from sqlglot import exp, parse_one

expression_tree = parse_one("SELECT a FROM x")

def transformer(node):
    if isinstance(node, exp.Column) and node.name == "a":
        return parse_one("FUN(a)")
    return node

transformed_tree = expression_tree.transform(transformer)

maxjustus

require "minitest/autorun"
require 'yaml'

`make test-render`

def flatten_hash(object, current_path = [], result = {})
  case object
  when Hash
    object.each do |key, value|
      flatten_hash(value, current_path + [key], result)

maxjustus

./s5cmd ls s3://mybucket/* | parallel --pipe -N 1000 'clickhouse client --query "insert into some_table FORMAT TSVRaw settings async_insert=1"'

maxjustus

sed 's/,/,:/g' my_file.csv | column -t -s:

maxjustus

#!/usr/bin/env ruby

require 'bundler/inline'

begin
  gemfile do
    source 'https://rubygems.org'
    gem 'tty-prompt', '~> 0.23.1', require: false
  end
rescue => e