frankyaorenjie

ALTER TABLE dashboard.search_stats_hourly_2018
PARTITION BY RANGE ( MONTH(date_hour) ) (
PARTITION p01 VALUES LESS THAN (2),
PARTITION p02 VALUES LESS THAN (3),
PARTITION p03 VALUES LESS THAN (4),
PARTITION p04 VALUES LESS THAN (5),
PARTITION p05 VALUES LESS THAN (6),
PARTITION p06 VALUES LESS THAN (7),
PARTITION p07 VALUES LESS THAN (8),
PARTITION p08 VALUES LESS THAN (9),

frankyaorenjie

bq
===
SELECT
  *
FROM
  [data-platform-163807:visearch_stats.access_logs_stats_hourly]
where date_hour >= '2014-01-01 00:00:00' and date_hour < '2015-01-01 00:00:00'

image
====

frankyaorenjie

Error: uninitialized constant Formulary::FormulaNamespacefe4ce29a01455f41d6d0b08c39f76615::Octave::DevelopmentTools
Please report this bug:
   https://git.io/brew-troubleshooting
/usr/local/Library/Taps/homebrew/homebrew-science/octave.rb:31:in `<class:Octave>’
/usr/local/Library/Taps/homebrew/homebrew-science/octave.rb:1:in `load_formula’
/usr/local/Library/Homebrew/formulary.rb:21:in `module_eval’
/usr/local/Library/Homebrew/formulary.rb:21:in `load_formula’
/usr/local/Library/Homebrew/formulary.rb:38:in `load_formula_from_path’
/usr/local/Library/Homebrew/formulary.rb:87:in `load_file’
/usr/local/Library/Homebrew/formulary.rb:78:in `klass’

frankyaorenjie

sudo chown -R $(whoami) /usr/local
brew update && brew upgrade # it will take some time
brew cask install xquartz
brew install octave # it will take a lot of time…

frankyaorenjie

set ulimit
set sysctl.conf
apt-get install build-essential
sudo apt-get update
sudo apt-get install -y ruby2.3-dev

frankyaorenjie

ulimit
vm.max_map_count via sysctl to be at least 262144.
sudo swapoff -a

http.cors.enabled : true
http.cors.allow-origin : "*"
index.number_of_replicas: 0

curl -XPUT 'localhost:9200/_template/logstash_template' -d ' 
{ 

frankyaorenjie

#V2 is branch name
#upstream is remote name
git push upstream HEAD:V2

frankyaorenjie

SELECT
  table_id,
  SUM(size_bytes)/POW(10,9) AS size
FROM
  `data-platform-163807.raw_logs.__TABLES__`
WHERE
  table_id LIKE '%20190101'
GROUP BY
  table_id
ORDER BY

frankyaorenjie

# parallel download
cat /tmp/urls.txt | parallel -j 100 wget --timeout=2 -q {}

frankyaorenjie

// read big file
split -l 500000 query_log_distinct_6m.csv split_files/