测试scrapy-cluster需要的环境安装脚本，一键安装openjdk, zookeeper, kafka, redis。 redis是需要本地编译的，请提前安装gcc

preinstall-scrapy-cluster.py

#!/usr/bin/env bash

if [ -n "${SUDO}" ]; then
    echo "ERROR: Please run as root"
    exit 1
fi

# === pip install ===
curl https://bootstrap.pypa.io/get-pip.py | python


# === supervisor install ===
pip install supervisor
systemctl start supervisord


# === jdk install ===
yum install java
cat <<EOF >> /etc/profile

export JAVA_HOME=/usr/lib/jvm/jre-openjdk
export CLASSPATH=/usr/lib/jvm/jre-openjdk/lib
export PATH=\$JAVA_HOME/bin:\$PATH
EOF
source /etc/profile


# === zookeeper install ===
ZOOKEEPER-VERSION="3.4.9"
wget http://mirrors.cnnic.cn/apache/zookeeper/zookeeper-${ZOOKEEPER-VERSION}/zookeeper-${ZOOKEEPER-VERSION}.tar.gz
tar -zxvf zookeeper-${ZOOKEEPER-VERSION}.tar.gz
mv zookeeper-${ZOOKEEPER-VERSION}/ /opt/zookeeper-${ZOOKEEPER-VERSION}/
cat <<EOF > /etc/supervisord.d/zookeeper.ini
[program:zookeeper]
command=/opt/zookeeper-${ZOOKEEPER-VERSION}/bin/zkServer.sh start-foreground
directory=/opt/zookeeper-${ZOOKEEPER-VERSION}
user=root
autostart=true
autorestart=true
startsecs=3
stdout_logfile=/var/log/supervisor/stdout.zookeeper.log
stderr_logfile=/var/log/supervisor/stderr.zookeeper.log
redirect_stderr=true
environment= JAVA_HOME=/opt/jdk1.8.0_101
EOF
supervisorctl reread
supervisorctl add zookeeper


# === kafka install ===
KAFKA-MAIN-VERSION="2.11"
KAFKA-LITTLE-VERSION="0.10.1.0"
KAFKA-LONG-VERSION="${KAFKA-MAIN-VERSION}-${KAFKA-LITTLE-VERSION}"
wget http://mirrors.cnnic.cn/apache/kafka/${KAFKA-LITTLE-VERSION}/kafka_${KAFKA-LONG-VERSION}.tgz
tar -zxvf kafka_${KAFKA-LONG-VERSION}.tgz
mv kafka_${KAFKA-LONG-VERSION}/ /opt/kafka_${KAFKA-LONG-VERSION}/
cat <<EOF > /etc/supervisord.d/kafka.ini
[program:kafka]
command=/opt/kafka_${KAFKA-LONG-VERSION}/bin/kafka-server-start.sh /opt/kafka_${KAFKA-LONG-VERSION}/config/server.properties
directory=/opt/kafka_${KAFKA-LONG-VERSION}
user=root
autostart=true
autorestart=true
startsecs=3
stdout_logfile=/var/log/supervisor/stdout.kafka.log
stderr_logfile=/var/log/supervisor/stderr.kafka.log
redirect_stderr=true
environment= JAVA_HOME=/opt/jdk1.8.0_101
EOF
supervisorctl reread
supervisorctl add kafka


# === redis install ===
REDIS-VERSION="3.2.8"
REDIS_PORT=6379
REDIS_LOG_FILE="/var/log/redis_${REDIS_PORT}.log"
REDIS_PID_FILE="/var/run/redis_${REDIS_PORT}.pid"
REDIS_DATA_DIR="/var/lib/redis/${REDIS_PORT}"
wget http://download.redis.io/releases/redis-${REDIS-VERSION}.tar.gz
cd redis-${REDIS-VERSION}/
make PREFIX=/opt/redis-${REDIS-VERSION} install
mkdir /opt/redis-${REDIS-VERSION}/conf.d
read -r REDIS_SED_EXPR <<-EOF
s#^port [0-9]{4}\$#port ${REDIS_PORT}#; \
s#^logfile .+\$#logfile ${REDIS_LOG_FILE}#; \
s#^dir .+\$#dir ${REDIS_DATA_DIR}#; \
s#^pidfile .+\$#pidfile ${REDIS_PID_FILE}#; \
s#^daemonize no\$#daemonize yes#;
EOF
sed -r "$REDIS_SED_EXPR" redis.conf  >> /opt/redis-${REDIS-VERSION}/conf.d/${REDIS_PORT}.conf
cd ~ && rm -rf redis-${REDIS-VERSION}/
cat <<EOF > /etc/supervisord.d/redis.ini
[program:redis]
command=/opt/redis-${REDIS-VERSION}/bin/redis-server /opt/redis-${REDIS-VERSION}/conf.d/${REDIS_PORT}.conf
directory=/opt/redis-${REDIS-VERSION}
user=root
autostart=true
autorestart=true
startsecs=3
stdout_logfile=/var/log/supervisor/stdout.redis.log
stderr_logfile=/var/log/supervisor/stderr.redis.log
redirect_stderr=true
EOF
supervisorctl reread
supervisorctl add redis