Installation script for scrapyd

install_scrapyd.sh

#
# Installation script for scrapyd on Debian/Ubuntu
# http://scrapyd.readthedocs.org/
#
# - Latest scrapyd from Scrapy package repository
# - Password protection with http basic auth
# - HTTPS with self-signed certificate
# - Works on Amazon EC2
#
FQDN=scrapy.example.com
PASSWORD=secretpass

# Install latest scrapyd
apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 627220E7
echo 'deb http://archive.scrapy.org/ubuntu scrapy main' >/etc/apt/sources.list.d/scrapy.list
apt-get update
apt-get install -y scrapyd

# Configure scrapyd
cat >/etc/scrapyd/conf.d/local <<EOF
[scrapyd]
bind_address = 127.0.0.1
EOF

# Create self-signed certificate for https
SSL=/etc/ssl/private
openssl req -nodes -newkey rsa:4096 -keyout $SSL/cert.key -out $SSL/cert.csr -subj "/CN=$FQDN"
openssl x509 -req -days 3650 -in $SSL/cert.csr -signkey $SSL/cert.key -out $SSL/cert.crt

# Setup webserver front-end
apt-get install -y nginx-light apache2-utils
cat >/etc/nginx/sites-enabled/default <<EOF
server {
  listen 6843 default_server;
  ssl on;
  ssl_certificate /etc/ssl/private/cert.crt;
  ssl_certificate_key /etc/ssl/private/cert.key;
  ssl_session_timeout 5m;
  ssl_protocols SSLv3 TLSv1 TLSv1.1 TLSv1.2;
  ssl_ciphers "HIGH:!aNULL:!MD5 or HIGH:!aNULL:!MD5:!3DES";
  ssl_prefer_server_ciphers on;

  location / {
    proxy_pass http://localhost:6800/;
    proxy_redirect off;
    proxy_set_header Authorization "";
    auth_basic "Restricted";
    auth_basic_user_file /etc/nginx/htpasswd;
  }
}
EOF
htpasswd -b -c /etc/nginx/htpasswd deploy "$PASSWORD"

# Restart and get ready
/etc/init.d/scrapyd restart
/etc/init.d/nginx restart