dontstopbelieveing

Chapter 2: Installing Kafka

Hardware Selection

• Disk Throughput is important, SSD's can be a better choice
• Disk Capacity determined by how many messages need to be retained at any time + 10% overhead
  • Total traffic for cluster can be balanced by having multiple partitions per topic
• Memory CPU no special requirements
• Network - one producer can have multiple consumers, this can cause inbound and outbound traffic imbalance
• How many brokers?
  • determined by disk capacity, network bandwidth and replication factor
• For example if each broker has storage 2 TB and cluster needs to retain 10TB data, we need 5 brokers, if RF is 1 we need 10 brokers, if one broker's bandwidth is at 80% we may need another broker etc.

dontstopbelieveing

#Slave Lag

#!/usr/bin/bash
log_dir=/logs/

#Slave lag
mysql --login-path=mysqlconn   -e "show slave status \G" | grep Seconds| awk '{ print $2 }' | while IFS= read -r line; do printf '%s\t%s\n' "$(date '+%Y-%m-%d  %H:%M:%S')" "$line"; done | tee -a $log_dir/slave_lag

#Processlist
mysql --login-path=mysqlconn   -e "show slave status \G" | grep Seconds| awk '{ print $2 }' | while IFS= read -r line; do printf '%s\t%s\n' "$(date '+%Y-%m-%d  %H:%M:%S')" "$line"; done | tee -a $log_dir/slave_lag

dontstopbelieveing

#!/usr/bin/bash

#export PATH=$PATH:/x/home/mysql/product/mysql/bin/mysql
#mysql --login-path=mysqlconn --batch --skip-column-names -e "select * from information_schema.processlist where user not in ('root','repl','system user') and host like '10.72.12.201%'" | while IFS= read -r line; do printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$line"; done | tee -a $log_dir/connection_metrics.log

mysql --login-path=mysqlconn --batch --skip-column-names -e "select now(),count(*) from information_schema.processlist where user not in ('root','repl','system user');" | tee -a /tmp/connection_metrics.log

dontstopbelieveing

select * from (SELECT a.manifest_label,a.cos,x.status FROM manifest_approval a, manifest_approval_history h, manifest_approval_action x where manifest_label in ('commercesetupnodeweb-1.2.0_20190924210218860', 'commercesetupnodeweb-1.2.0_20190925124406157', 'commercesetupnodeweb-1.1.3_2019092415463768', 'commercesetupnodeweb-1.1.3_20190924144432630', 'commercesetupnodeweb-1.1.3_20190923164432256', 'commercesetupnodeweb-1.1.3_20190923154136989', 'commercesetupnodeweb-1.1.3_20190922225234618', 'commercesetupnodeweb-1.1.3_20190922233835112', 'commercesetupnodeweb-1.1.2_20190922145056771', 'commercesetupnodeweb-1.1.2_20190921220059754', 'commercesetupnodeweb-1.1.2_20190920220652189', 'commercesetupnodeweb-1.1.2_20190918124024272', 'commercesetupnodeweb-1.1.2_20190917132435888', 'commercesetupnodeweb-1.1.2_20190916164942679', 'commercesetupnodeweb-1.1.2_20190916165027142', 'commercesetupnodeweb-1.1.2_20190916134019187', 'commercesetupnodeweb-1.1.1_2019091310434924', 'commercesetupnodeweb-1.1.1_2019091216405973', 'c

dontstopbelieveing

{
  "Debug": false,
  "EnableSyslog": false,
  "ListenAddress": ":3000",
  "MySQLTopologyUser": "orc_client_user",
  "MySQLTopologyPassword": "orc_client_password",
  "MySQLTopologyCredentialsConfigFile": "/etc/orchestrator-topology.cnf",
  "MySQLTopologySSLPrivateKeyFile": "",
  "MySQLTopologySSLCertFile": "",
  "MySQLTopologySSLCAFile": "",

dontstopbelieveing

#!/usr/bin/python
import random
from datetime import datetime

import MySQLdb as mysql

host = 
user = 
password = 
db = 

dontstopbelieveing

./sysbench --test=tests/db/oltp.lua \
--mysql-host=$host \
--mysql-port=3306 \
--mysql-user=reinvent \
--mysql-password=reinvent \
--mysql-db=$db \
--mysql-table-engine=innodb \
--oltp-test-mode=complex \
--oltp-read-only=on \
--oltp-table-size=1000000 \

dontstopbelieveing

-- In absence of sys schema
SELECT table_name,
       ROUND(SUM(IF(compressed_size = 0, 16384, compressed_size)) / 1073741824, 2)  AS allocated_GB,
       ROUND(SUM(data_size) / 1073741824, 2)  AS data_GB,
       COUNT(page_number) AS pages,
       COUNT(IF(is_hashed = 'YES', 1, NULL)) AS pages_hashed,
       COUNT(IF(is_old = 'YES', 1, NULL)) AS pages_old,
       ROUND(SUM(number_records)/COUNT(DISTINCT index_name)) AS rows_cached 
 FROM information_schema.innodb_buffer_page
 WHERE table_name = "`db_name`.`table_name`"

dontstopbelieveing

 --Creating the database 
DROP DATABASE online_store;
CREATE DATABASE online_store;

--Creating supporting tables
USE online_store;
CREATE TABLE brands (
    id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
    name VARCHAR(255) NOT NULL
);