Eugene Klimov Slach

steps to reproduce

docker-compose down
docker-compose run clickhouse

For try to reproduce run following command in bash

docker-compose down && docker-compose up -d clickhouse && sleep 5 && docker-compose exec clickhouse bash ./reproduce.sh

Обидно, когда не получается вместе использовать два таких достойных продукта

Суть проблемы: При выгрузке данных из Metrica Logs API, генерируются TSV файлы которые потом не распознаются парсером Clickhouse для TabSeparatedWithNames формата и соответвенно не могут напрямую быть загружены в локальный Clickhouse

вот основные ошибки — одинарные ковычки, которые обрамляют строки в комплексных типах Array (String) эскейпятся через
пример это поля ym:s:goalsDateTime и ym:s:goalsOrder

	2020.08.24 13:44:17.013175 [ 1 ] {} <Information> : Starting ClickHouse 20.4.9.110 with revision 54434, PID 1
	2020.08.24 13:44:17.019023 [ 1 ] {} <Information> Application: starting up
	2020.08.24 13:44:17.028135 [ 1 ] {} <Debug> Application: rlimit on number of file descriptors is 1048576
	2020.08.24 13:44:17.028193 [ 1 ] {} <Debug> Application: Initializing DateLUT.
	2020.08.24 13:44:17.028247 [ 1 ] {} <Debug> Application: Setting up /var/lib/clickhouse/tmp/ to store temporary data in it
	2020.08.24 13:44:17.028938 [ 1 ] {} <Debug> Application: Configuration parameter 'interserver_http_host' doesn't exist or exists and empty. Will use 'chi-test-cluster-for-alerts-default-0-0-0.chi-test-cluster-for-alerts-default-0-0.test.svc.cluster.local' as replica host.
	2020.08.24 13:44:17.033778 [ 1 ] {} <Debug> ConfigReloader: Loading config '/etc/clickhouse-server/users.xml'
	2020.08.24 13:44:17.041703 [ 1 ] {} <Warning> Access(disk): File /var/lib/clickhouse/access/users.list doesn't exist
	2020.08.24 13:44:17.041877 [ 1 ]

	* ==> Docker <==
	* -- Logs begin at Tue 2019-12-24 09:36:37 UTC, end at Tue 2019-12-24 09:41:14 UTC. --
	* Dec 24 09:36:55 minikube dockerd[2452]: time="2019-12-24T09:36:55.226032614Z" level=info msg="loading plugin "io.containerd.runtime.v2.task"..." type=io.containerd.runtime.v2
	* Dec 24 09:36:55 minikube dockerd[2452]: time="2019-12-24T09:36:55.226276799Z" level=info msg="loading plugin "io.containerd.monitor.v1.cgroups"..." type=io.containerd.monitor.v1
	* Dec 24 09:36:55 minikube dockerd[2452]: time="2019-12-24T09:36:55.227344467Z" level=info msg="loading plugin "io.containerd.service.v1.tasks-service"..." type=io.containerd.service.v1
	* Dec 24 09:36:55 minikube dockerd[2452]: time="2019-12-24T09:36:55.227435429Z" level=info msg="loading plugin "io.containerd.internal.v1.restart"..." type=io.containerd.internal.v1
	* Dec 24 09:36:55 minikube dockerd[2452]: time="2019-12-24T09:36:55.227552987Z" level=info msg="loading plugin "io.containerd.grpc.v1.containers"..." type=io.containerd.grpc.v1
	* Dec 24 09:36:55

	package main;
	import (
	grameworklog "github.com/apex/log"
	grameworklog_json "github.com/apex/log/handlers/json"
	"github.com/gramework/gramework"
	)

	func InitGramework() (web *gramework.App) {
	web = gramework.New()
	logger := &grameworklog.Logger{

	root@mysql-proxy:/home/vagrant# cat /var/lib/proxysql/proxysql.log
	2018-04-01 14:27:30 [INFO] ProxySQL version 1.4.6-1.1
	2018-04-01 14:27:30 [INFO] Detected OS: Linux mysql-proxy 4.4.0-116-generic #140-Ubuntu SMP Mon Feb 12 21:23:04 UTC 2018 x86_64
	2018-04-01 14:27:30 [INFO] Starting ProxySQL
	2018-04-01 14:27:30 [INFO] Sucessfully started
	2018-04-01 14:27:30 [INFO] Angel process started ProxySQL process 7834
	Standard ProxySQL Cluster rev. 0.1.0702 -- ProxySQL_Cluster.cpp -- Fri Mar 9 07:46:21 2018
	Standard ProxySQL Statistics rev. 1.4.1027 -- ProxySQL_Statistics.cpp -- Fri Mar 9 07:46:21 2018
	Standard ProxySQL HTTP Server Handler rev. 1.4.1031 -- ProxySQL_HTTP_Server.cpp -- Fri Mar 9 07:46:21 2018
	Standard ProxySQL Admin rev. 0.2.0902 -- ProxySQL_Admin.cpp -- Fri Mar 9 07:46:21 2018

	*.log
	.vagrant/
	.idea/
	*.exe
	*.dll
	*.so
	*.dylib
	*.pcap

	id_rsa

	config.yml
	anomalies/
	src/github.com
	src/gopkg.in
	src/golang.org
	src/vendor
	vendor/
	*.log
	pkg/
	dist/

	.vagrant
	*.log