environments.json

{
   "secret_key":
   "akismet_api_key":
   "db_default_user":
   "db_default_password":
   "db_default_host":
   "db_default_port":
}

fabfile.py

# -*- coding: utf-8 -*-

import os
import socket
import getpass

from fabric.api import local, run, sudo, cd, prefix, put
from fabric.state import env
from fabric.contrib.files import sed, exists, append, contains, uncomment
from fabric.network import join_host_strings, normalize
from fabric.context_managers import settings

"""
Fab settings
"""
env.hosts = ['djangoproject.vir']
env.git_code_url = 'git://github.com/django-ja/djangoproject.jp.git'
env.git_docs_url = 'git://github.com/django-docs-ja/django-docs-ja.git'
env.mysqluser = env.user
env.mysqlpassword = 'enterpassword'
env.mysqlport = '3308'
env.dbname = 'djangoprojectjp'
env.secret_key = 'entersecretkey'
env.akismet_api_key = 'enterakismetkey'

def change_ssh_port(after=22):
    """
    For security changing the default ssh port.
    """
    host = normalize(env.host_string)[1]
    before = str(env.port)

    host_string=join_host_strings(env.user,host,before)
    with settings(host_string=host_string, user=env.user):
        print env.host, "CHANGING SSH PORT TO: "+str(after)
        sed('/etc/ssh/sshd_config','Port '+ str(before),'Port '+str(after),use_sudo=True)
        sudo('/etc/init.d/ssh restart')
        return True

def upload_ssh_key(rollback=False):
    """
    Upload your ssh key for passwordless logins
    """
    auth_keys = '/home/%s/.ssh/authorized_keys' % env.user
    if not rollback:
        local_user = getpass.getuser()
        host = socket.gethostname()
        u = '@'.join([local_user,host])
        u = 'ssh-key-uploaded-%s'% u
        with cd('/home/%s' % env.user):
            if not exists('.ssh'):
                sudo('mkdir .ssh')
           
        # determine local .ssh dir
        home = os.path.expanduser('~')
        ssh_key = None
        ssh_dsa = os.path.join(home,'.ssh/id_dsa.pub')
        ssh_rsa = os.path.join(home,'.ssh/id_rsa.pub')
        if os.path.exists(ssh_dsa):
            ssh_key = ssh_dsa
        elif os.path.exists(ssh_rsa):
            ssh_key = ssh_rsa
    
        if ssh_key:
            ssh_file = open(ssh_key,'r').read()
            
            print env.host, "UPLOADING SSH KEY"
            append(auth_keys,ssh_file, use_sudo=True)
        return
    else:
        return

def restrict_ssh():
    """
    Restrict password login, and root login.
    """
    sshd_config = '/etc/ssh/sshd_config'
    if not exists('/home/%s/.ssh/authorized_keys' % env.user):
        print env.host, 'You need to upload_ssh_key first.'
        return False
    sed(sshd_config, 'PermitRootLogin yes', 'PermitRootLogin no', use_sudo=True)
    uncomment(sshd_config,'#(\s?)PermitRootLogin(\s*)no',use_sudo=True)
    sed(sshd_config, 'PermitEmptyPassword yes', 'PermitEmptyPassword no', use_sudo=True)
    uncomment(sshd_config,'#(\s?)PermitEmptyPassword(\s*)no',use_sudo=True)
    sed(sshd_config, 'PasswordAuthentication yes', 'PasswordAuthentication no', use_sudo=True)
    uncomment(sshd_config,'#(\s?)PasswordAuthentication(\s*)no',use_sudo=True)

    sudo('/etc/init.d/ssh restart')
        
def setup_ufw_rules(sshport):
    """
    Setup ufw app rules.

    """
    firewall_rules = ['default deny', 'allow 80/tcp', 'allow 443/tcp', 'allow %s/tcp' % sshport]

    print 'deleting all rules'
    sudo('ufw reset')

    print 'enable ufw'
    sudo('ufw enable')

    for rule in firewall_rules:
        with settings(warn_only=True):
            print 'ufw', rule
            sudo('ufw %s'% rule)
        
    output = sudo('ufw reload')
    print output

def disable_services():
    """
    For performance, disable unnecesaly daemon.
    """
    disables = ['acpid', 'apmd', 'atd', 'dns-clean', 'pppd-dns']

    for d in disables:
        if exists('/etc/init.d/%s', d):
            sudo('update-rc.d %s disable' % d)

def install_packages():
    """
    Install a set of baseline packages and configure where necessary
    """
    base_packages = ['nginx', 'mysql-server', 'python-setuptools', 'python-dev', 'libmysqlclient-dev', 'build-essential', 'git']
    python_packages = ['virtualenv', 'pip']

    sudo('apt-get update')

    for p in base_packages:
        sudo('apt-get install %s' % p)

    with cd('/tmp'):
        for p in python_packages:
            sudo('easy_install %s' % p)

def deploy_code():
    """
    Update code on the servers from Git.
    """
    if not exists('~/djangoproject.jp/'):
        with cd('~'):
            print "cloning djangoproject.jp: %s" % env.git_code_url
            run('git clone %s' % env.git_code_url)
    with cd('/home/%s/djangoproject.jp/' % env.user):
        run('git fetch && git reset --hard origin/master')

def update_virtualenv(env_name, packages, reqs=None):
    """
    Update virtualenv for the project, and update depencies.
    """
    if not exists('~/envs/%s' % env_name):
        if not exists('~/envs'):
            run('mkdir envs')
        with cd('~/envs'):
            run('virtualenv %s' % env_name)
    with prefix('source ~/envs/%s/bin/activate' % env_name):
        run('export PIP_RESPECT_VIRTUALENV=true')
        for p in packages:
            sudo('pip install "%s"' % p)
        if reqs:
            sudo('pip install -r %s' % reqs)

def deploy_doc(doc_version):
    """
    Update docs on the servers from Git.
    """
    with prefix('source ~/envs/djangoproject.jp/bin/activate'):
        run('export PIP_RESPECT_VIRTUALENV=true')
        # ドキュメントのバージョンに応じたvirtualenvを作成
        if doc_version == "1.0":
            update_virtualenv('1.0', ['Sphinx==0.4.3'])
        else:
            sudo('pip install Sphinx')

    if not exists('~/doc/%s/' % doc_version):
        if not exists('~/doc/'):
            run('mkdir doc')
        with cd('~/doc'):
            run('git clone %s' % (env.git_docs_url))
            run('mv django-docs-ja %s' % doc_version)
    with cd('~/doc/%s' % doc_version):
        with prefix('source ~/envs/%s/bin/activate' % doc_version):
            run('git fetch && git reset --hard %s' % doc_version)
            run('make html')

def setup_mysql():
    """
    Setup mysql
    """
    # my.cnfの設定
    sudo('/etc/init.d/mysql stop')
    put('./my.cnf', '/etc/mysql/my.cnf', use_sudo=True)
    sed('/etc/mysql/my.cnf', 'port = 3306', 'port = %s' % env.mysqlport, use_sudo=True)
    sudo('mkdir -p -m 660 /var/lib/mysql/backup')
    sudo('mv /var/lib/mysql/ib_logfile* /var/lib/mysql/backup')
    sudo('mv /var/lib/mysql/ibdata* ./backup')
    sudo('/etc/init.d/mysql start')

    # アクセス用ユーザの作成
    create_user_sql = "grant select,insert,delete,update,create,drop,file,alter,index on *.* to '%s'@'localhost' identified by '%s';" % (env.mysqluser, env.mysqlpassword)
    run('echo "%s" | mysql --user=root --password' % create_user_sql)

    run('mysqladmin -u %s -p%s --default-character-set=utf8 create %s' % (env.mysqluser, env.mysqlpassword, env.dbname))

def deploy_static():
    with cd('djangoproject.jp'):
        with prefix('source ~/envs/djangoproject.jp/bin/activate'):
            run('mkdir -p static')
            run('python manage.py collectstatic -v0 --noinput')

def setup_gunicorn():
    """
    Setup gunicorn, and start daemon.
    """
    run('source ~/envs/djangoproject.jp/bin/activate && python djangoproject.jp/manage.py run_gunicorn -w 1 -b 127.0.0.1:5000 -D && ps -aux | grep gunicorn')
    run('ps -aux | grep gunicorn')

def setup_nginx():
    """
    Setup nginx.
    """
    # plain_defaultからdefaultを生成しアップロード
    local('cp ./plain_default ./default')
    local('sed -i -e "s/{{ username }}/%s/g" ./default' % env.user)
    put('./default', '/etc/nginx/sites-available/default', use_sudo=True)

    sudo('nginx -t')
    sudo('/etc/init.d/nginx reload')

def setup_environments():
    """
    Update enviroments.json which to set secret informetions.
    """
    with cd('~/djangoproject.jp'):
        # environmentsがすでに存在している場合は削除
        if exists('environments.json'):
            run('rm environments.json')

        put(os.path.dirname(__file__)+'/environments.json', '/home/%s/djangoproject.jp/environments.json' % env.user)
        sed('environments.json', '"secret_key":', '"secret_key":"%s",' % env.secret_key)
        sed('environments.json', '"akismet_api_key":', '"akismet_api_key":"%s",' % env.akismet_api_key)
        sed('environments.json', '"db_default_user":', '"db_default_user":"%s",' % env.mysqluser)
        sed('environments.json', '"db_default_password":', '"db_default_password":"%s",' % env.mysqlpassword)
        sed('environments.json', '"db_default_host":', '"db_default_host":"localhost",')
        sed('environments.json', '"db_default_port":', '"db_default_port":"%s"' % env.mysqlport)

def setup_logrotate():
    put('./logrotate.conf', '/etc/logrotate.conf', use_sudo=True)

def managepy(cmd):
    """
    Helper: run a management command remotely.
    """
    with prefix('source ~/envs/djangoproject.jp/bin/activate'):
        with cd('~/djangoproject.jp'):
            run('python manage.py %s' % cmd)

def syncdb():
    """
    Run syncdb.
    """
    managepy('syncdb')

def base_deploy(sshport):
    """
    Base deploy: secure login, firewall, disable unnecessaly searvicies.
    """
    change_ssh_port(sshport)
    upload_ssh_key()
    restrict_ssh()
    setup_ufw_rules(sshport)
    disable_services()
    sudo('reboot')

def full_deploy():
    """
    Full deploy: new code, update dependencies, migrate, and restart services.
    """
    install_packages()
    deploy_code()
    setup_environments()
    update_virtualenv('djangoproject.jp', ['MySQL-python'], '~/djangoproject.jp/requirement.txt')
    deploy_doc('1.0')
    setup_mysql()
    setup_logrotate()
    syncdb()
    deploy_static()
    setup_gunicorn()
    setup_nginx()

logrotate.conf

# see "man logrotate" for details
# rotate log files weekly
weekly

# keep 4 weeks worth of backlogs
rotate 4

# create new (empty) log files after rotating old ones
create

# uncomment this if you want your log files compressed
#compress

# packages drop log rotation information into this directory
include /etc/logrotate.d

# no packages own wtmp, or btmp -- we'll rotate them here
/var/log/wtmp {
    missingok
    monthly
    create 0664 root utmp
    rotate 1
}

/var/log/btmp {
    missingok
    monthly
    create 0660 root utmp
    rotate 1
}

/var/log/nginx/*.log {
    daily
    missingok
    rotate 90
    compress
    ifempty
    sharedscripts
    postrotate
        [ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
    endscript

# system-specific logs may be configured here

my.cnf

# This is for a large system with memory of 1G-2G where the system runs mainly
# MySQL.
#
# In this file, you can use all long options that a program supports.
# If you want to know which options a program supports, run the program
# with the "--help" option.

# The following options will be passed to all MySQL clients
[client]
port = 3306
socket = /var/run/mysqld/mysqld.sock
default-character-set = utf8

# Here follows entries for some specific programs

# The MySQL server
[mysqld]
port = 3306
socket = /var/run/mysqld/mysqld.sock
skip-external-locking
key_buffer = 128M
max_allowed_packet = 1M
table_cache = 512
sort_buffer_size = 1M
read_buffer_size = 1M
read_rnd_buffer_size = 8M
myisam_sort_buffer_size = 1M
thread_cache_size = 8
query_cache_size = 32M
# Try number of CPU's*2 for thread_concurrency
thread_concurrency = 8
character-set-server = utf8
skip-character-set-client-handshake

# Don't listen on a TCP/IP port at all. This can be a security enhancement,
# if all processes that need to connect to mysqld run on the same host.
# All interaction with mysqld must be made via Unix sockets or named pipes.
# Note that using this option without enabling named pipes on Windows
# (via the "enable-named-pipe" option) will render mysqld useless!
# 
#skip-networking

# Replication Master Server (default)
# binary logging is required for replication
log-bin=mysql-bin
log-bin-index=mybin.index
max-binlog-size=1G
expire-logs-days=7

# required unique id between 1 and 2^32 - 1
# defaults to 1 if master-host is not set
# but will not function as a master if omitted
server-id       = 1

# Replication Slave (comment out master section to use this)
#
# To configure this host as a replication slave, you can choose between
# two methods :
#
# 1) Use the CHANGE MASTER TO command (fully described in our manual) -
#    the syntax is:
#
#    CHANGE MASTER TO MASTER_HOST=<host>, MASTER_PORT=<port>,
#    MASTER_USER=<user>, MASTER_PASSWORD=<password> ;
#
#    where you replace <host>, <user>, <password> by quoted strings and
#    <port> by the master's port number (3306 by default).
#
#    Example:
#
#    CHANGE MASTER TO MASTER_HOST='125.564.12.1', MASTER_PORT=3306,
#    MASTER_USER='joe', MASTER_PASSWORD='secret';
#
# OR
#
# 2) Set the variables below. However, in case you choose this method, then
#    start replication for the first time (even unsuccessfully, for example
#    if you mistyped the password in master-password and the slave fails to
#    connect), the slave will create a master.info file, and any later
#    change in this file to the variables' values below will be ignored and
#    overridden by the content of the master.info file, unless you shutdown
#    the slave server, delete master.info and restart the slaver server.
#    For that reason, you may want to leave the lines below untouched
#    (commented) and instead use CHANGE MASTER TO (see above)
#
# required unique id between 2 and 2^32 - 1
# (and different from the master)
# defaults to 2 if master-host is set
# but will not function as a slave if omitted
#server-id       = 2
#
# The replication master for this slave - required
#master-host     =   <hostname>
#
# The username the slave will use for authentication when connecting
# to the master - required
#master-user     =   <username>
#
# The password the slave will authenticate with when connecting to
# the master - required
#master-password =   <password>
#
# The port the master is listening on.
# optional - defaults to 3306
#master-port     =  <port>
#
# binary logging - not required for slaves, but recommended
#log-bin=mysql-bin

# Point the following paths to different dedicated disks
#tmpdir         = /tmp/         
#log-update     = /path-to-dedicated-directory/hostname

# Uncomment the following if you are using BDB tables
#bdb_cache_size = 384M
#bdb_max_lock = 100000

# Uncomment the following if you are using InnoDB tables
#innodb_data_home_dir = /var/lib/mysql/
#innodb_data_file_path = ibdata1:2000M;ibdata2:10M:autoextend
#innodb_log_group_home_dir = /var/lib/mysql/
#innodb_log_arch_dir = /var/lib/mysql/
# You can set .._buffer_pool_size up to 50 - 80 %
# of RAM but beware of setting memory usage too high
innodb_buffer_pool_size = 256M
innodb_additional_mem_pool_size = 20M
# Set .._log_file_size to 25 % of buffer pool size
innodb_log_file_size = 64M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 1
innodb_lock_wait_timeout = 50
innodb_log_files_in_group = 2

[mysqldump]
quick
max_allowed_packet = 16M
default-character-set = utf8

[mysql]
no-auto-rehash
# Remove the next comment character if you are not familiar with SQL
#safe-updates
default-character-set = utf8

[isamchk]
key_buffer = 128M
sort_buffer_size = 1M
read_buffer = 1M
write_buffer = 1M

[myisamchk]
key_buffer = 128M
sort_buffer_size = 1M
read_buffer = 1M
write_buffer = 1M

[mysqlhotcopy]
interactive-timeout

plain_default

upstream djangoja-website {
    server 127.0.0.1:5000;
}

server {
    listen 80;
    server_name localhost;

    access_log /var/log/nginx/localhost.access.log;
    error_log /var/log/nginx/localhost.error.log debug;

    location ^~ /doc/ja/1.0 {
        alias /home/{{ username }}/doc/1.0/_build/html;
        index index.html;
        break;
    }

    location ^~ /m {
        alias /home/{{ username }}/djangoproject.jp/media;
        break;
    }

    location / {
        proxy_pass http://djangoja-website;
        break;
    }



}