iahmad-khan

C02W84XMHTD5:ucp iahmad$ kubectl get pods --namespace=kube-system 
NAME                                    READY   STATUS    RESTARTS   AGE
coredns-c4cffd6dc-nsd2k                 1/1     Running   0          1d
etcd-minikube                           1/1     Running   0          1d
kube-addon-manager-minikube             1/1     Running   0          1d
kube-apiserver-minikube                 1/1     Running   0          1d
kube-controller-manager-minikube        1/1     Running   0          1d
kube-dns-86f4d74b45-d5njm               3/3     Running   0          1d
kube-proxy-rt45q                        1/1     Running   0          1d

iahmad-khan

#/bin/bash

if [ $# -lt 3 ]
then
  echo "Usage: reindex.sh source_url target_url index [query]"
  echo "Example: reindex.sh http://source.com:9200 http://target.com:9200 products_v1 '{\"query\":{\"range\":{\"mod_date\":{\"from\":\"2015-12-03T20:00:00\"}}}}'"
  exit 1
fi

SOURCE_URL=$1

iahmad-khan

# example python script

# pip install requests before using requests
import requests
import json

HOSTNAME="your.elasticsearch.host.com" # hostname
PORT=9200 # port number
NODE_NAME="node001" # node to reroute to 

iahmad-khan

# example python script

# pip install requests before using requests
import requests
import json

HOSTNAME="your.elasticsearch.host.com" # hostname
PORT=9200 # port number
NODE_NAME="node001" # node to reroute to 

iahmad-khan

2 node Cluster:

    master node:  elasticsearch , logstash , kibana , niginx
    data node: elasticsearch worker


Installation:

  - all the above services are install using standard ubuntu package manager

iahmad-khan

[2018-01-26T16:19:18,565][DEBUG][logstash.pipeline        ] filter received {"event"=>{"@timestamp"=>2018-01-26T16:17:53.278Z, "offset"=>336914229, "source"=>"/usr/local/zeus/log/wwwint.log", "@version"=>"1", "message"=>"10.3.7.49 - - [26/Jan/2018:07:38:29 +0000] \"GET /solr/citations/query?q=AUTH:%22McEntyre+J%22&rows=0 HTTP/1.1\" 200 358 \"-\" \"Java/1.8.0_131\" ves-pg-3f:8983 0.008742 wwwint.ebi.ac.uk", "host"=>"www-lb3.ebi.ac.uk", "beat"=>{"hostname"=>"www-lb3.ebi.ac.uk", "name"=>"www-lb3.ebi.ac.uk", "version"=>"6.1.2"}, "tags"=>["beats_input_codec_plain_applied"], "prospector"=>{"type"=>"log"}}}
[2018-01-26T16:19:18,565][DEBUG][logstash.pipeline        ] filter received {"event"=>{"offset"=>336914410, "prospector"=>{"type"=>"log"}, "source"=>"/usr/local/zeus/log/wwwint.log", "@version"=>"1", "message"=>"10.49.1.53 - - [26/Jan/2018:07:38:29 +0000] \"GET /solr/citations/query?q=AUTH:%22McEntyre+J%22&rows=0 HTTP/1.1\" 200 358 \"-\" \"Java/1.8.0_74\" ves-pg-40:8983 0.009794 wwwint.ebi.ac.uk", "host"=>"www-lb

iahmad-khan

# Systemd unit file for tomcat
[Unit]
Description=Apache Tomcat Web Application Container
After=syslog.target network.target

[Service]
Type=forking

Environment=JAVA_HOME=/usr/lib/jvm/jre
Environment=CATALINA_PID=/opt/tomcat/temp/tomcat.pid

iahmad-khan

# Some notes from installing VirtualBox on CentOS 7. 
# These exact steps haven't been tested, as I ran them in a different order when manually installing.

# Install dependencies
yum -y install gcc make patch  dkms qt libgomp
yum -y install kernel-headers kernel-devel fontforge binutils glibc-headers glibc-devel

# Install VirtualBox
cd /etc/yum.repos.d
wget http://download.virtualbox.org/virtualbox/rpm/rhel/virtualbox.repo

iahmad-khan

cat ~/credentials/user
source ~/credentials/user
keystone discover
keystone catalog
keystone endpoint-get --service volume
keystone token-get --wrap 50

nova list
nova flavor-list
nova boot --image cirros-qcow2 --flavor 1 MyFirstInstance

iahmad-khan

// load error messages from a log into memory
// then interactively search for various patterns

// base RDD
val lines = sc.textFile("log.txt")

// transformed RDDs
val errors = lines.filter(_.startsWith("ERROR"))
val messages = errors.map(_.split("\t")).map(r => r(1))
messages.cache()