Skip to content

Instantly share code, notes, and snippets.

@loren138
Last active January 17, 2021 08:26
Show Gist options
  • Save loren138/27d9f53a4dfb3df206dd to your computer and use it in GitHub Desktop.
Save loren138/27d9f53a4dfb3df206dd to your computer and use it in GitHub Desktop.
MySQL Master Slave and EC2 Laravel Notes with Rocketeer
<?php
use Rocketeer\Services\Connections\ConnectionsHandler;
return [
// The name of the application to deploy
// This will create a folder of the same name in the root directory
// configured above, so be careful about the characters used
'application_name' => 'resources',
// Plugins
////////////////////////////////////////////////////////////////////
// The plugins to load
'plugins' => [// 'Rocketeer\Plugins\Slack\RocketeerSlack',
],
// Logging
////////////////////////////////////////////////////////////////////
// The schema to use to name log files
'logs' => function (ConnectionsHandler $connections) {
return sprintf('%s-%s-%s.log', $connections->getConnection(), $connections->getStage(), date('Ymd'));
},
// Remote access
//
// You can either use a single connection or an array of connections
////////////////////////////////////////////////////////////////////
// The default remote connection(s) to execute tasks on
'default' => ['production', 'production2'],
// The various connections you defined
// You can leave all of this empty or remove it entirely if you don't want
// to track files with credentials : Rocketeer will prompt you for your credentials
// and store them locally
'connections' => [
'production' => [
'host' => 'slave internal ip',
'username' => 'ec2-user',
'password' => '',
'key' => 'aws EC2 key',
'keyphrase' => '',
'agent' => '',
'db_role' => true,
],
'production2' => [
'host' => 'slave external ip',
'username' => 'ec2-user',
'password' => '',
'key' => 'awsEC2Key',
'keyphrase' => '',
'agent' => '',
'db_role' => false,
],
],
/*
* In most multiserver scenarios, migrations must be run in an exclusive server.
* In the event of not having a separate database server (in which case it can
* be handled through connections), you can assign a 'db_role' => true to the
* server's configuration and it will only run the migrations in that specific
* server at the time of deployment.
*/
'use_roles' => true,
// Contextual options
//
// In this section you can fine-tune the above configuration according
// to the stage or connection currently in use.
// Per example :
// 'stages' => array(
// 'staging' => array(
// 'scm' => array('branch' => 'staging'),
// ),
// 'production' => array(
// 'scm' => array('branch' => 'master'),
// ),
// ),
////////////////////////////////////////////////////////////////////
'on' => [
// Stages configurations
'stages' => [],
// Connections configuration
'connections' => [],
],
];
<?php
return [
// Tasks
//
// Here you can define in the `before` and `after` array, Tasks to execute
// before or after the core Rocketeer Tasks. You can either put a simple command,
// a closure which receives a $task object, or the name of a class extending
// the Rocketeer\Abstracts\AbstractTask class
//
// In the `custom` array you can list custom Tasks classes to be added
// to Rocketeer. Those will then be available in the command line
// with all the other tasks
//////////////////////////////////////////////////////////////////////
// Tasks to execute before the core Rocketeer Tasks
'before' => [
'setup' => [],
'deploy' => [],
'cleanup' => [],
],
// Tasks to execute after the core Rocketeer Tasks
'after' => [
'setup' => [],
'deploy' => [
'php artisan config:cache',
'php artisan route:cache',
'php artisan optimize'
],
'cleanup' => [],
],
// Custom Tasks to register with Rocketeer
'custom' => [],
];
<?php
return [
// Configurable paths
//
// Here you can manually set paths to some commands Rocketeer
// might try to use, if you leave those empty it will try to find them
// manually or assume they're in the root folder
//
// You can also add in this file custom paths for any command or binary
// Rocketeer might go looking for
////////////////////////////////////////////////////////////////////
// Path to the PHP binary
'php' => '',
// Path to Composer
'composer' => '',
// Path to the Artisan CLI
'artisan' => 'artisan',
];
<?php
return [
// Remote server
//////////////////////////////////////////////////////////////////////
// Variables about the servers. Those can be guessed but in
// case of problem it's best to input those manually
'variables' => [
'directory_separator' => '/',
'line_endings' => "\n",
],
// The number of releases to keep at all times
'keep_releases' => 4,
// Folders
////////////////////////////////////////////////////////////////////
// The root directory where your applications will be deployed
// This path *needs* to start at the root, ie. start with a /
'root_directory' => '/var/www/',
// The folder the application will be cloned in
// Leave empty to use `application_name` as your folder name
'app_directory' => '',
// A list of folders/file to be shared between releases
// Use this to list folders that need to keep their state, like
// user uploaded data, file-based databases, etc.
'shared' => [
'storage/logs',
'.env'
//'storage/sessions',
],
// Execution
//////////////////////////////////////////////////////////////////////
// If enabled will force a shell to be created
// which is required for some tools like RVM or NVM
'shell' => false,
// An array of commands to run under shell
'shelled' => ['which', 'ruby', 'npm', 'bower', 'bundle', 'grunt'],
// Enable use of sudo for some commands
// You can specify a sudo user by doing
// 'sudo' => 'the_user'
'sudo' => false,
// An array of commands to run under sudo
'sudoed' => [],
// Permissions$
////////////////////////////////////////////////////////////////////
'permissions' => [
// The folders and files to set as web writable
'files' => [
//'app/database/production.sqlite',
'storage',
'public',
],
// Here you can configure what actions will be executed to set
// permissions on the folder above. The Closure can return
// a single command as a string or an array of commands
'callback' => function ($task, $file) {
return [
sprintf('find %s -type f -exec chmod gu+w {} \;', $file),
sprintf('find %s -type d -exec chmod gu+w {} \;', $file),
sprintf('find %s -type d -exec chmod g+s {} \;', $file), //sprintf('chmod -R g+s %s', $file),
sprintf('chown -R ec2-user:www %s', $file),
];
},
],
];
<?php
return [
// SCM repository
//////////////////////////////////////////////////////////////////////
// The SCM used (supported: "git", "svn")
'scm' => 'git',
// The SSH/HTTPS address to your repository
// Example: https://github.com/vendor/website.git
'repository' => 'your git repo',
// The repository credentials : you can leave those empty
// if you're using SSH or if your repository is public
// In other cases you can leave this empty too, and you will
// be prompted for the credentials on deploy. If you don't want
// to be prompted (public repo, etc) set the values to null
'username' => '',
'password' => '',
// The branch to deploy
'branch' => 'master',
// Whether your SCM should do a "shallow" clone of the repository
// or not – this means a clone with just the latest state of your
// application (no history)
// If you're having problems cloning, try setting this to false
'shallow' => true,
// Recursively pull in submodules. Works only with GIT.
'submodules' => true,
];
<?php
return [
// Stages
//
// The multiples stages of your application
// if you don't know what this does, then you don't need it
//////////////////////////////////////////////////////////////////////
// Adding entries to this array will split the remote folder in stages
// Like /var/www/yourapp/staging and /var/www/yourapp/production
'stages' => [],
// The default stage to execute tasks on when --stage is not provided
// Falsey means all of them
'default' => '',
];
<?php
/*
* This file is part of Rocketeer
*
* (c) Maxime Fabre <[email protected]>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
use Rocketeer\Binaries\PackageManagers\Composer;
use Rocketeer\Tasks\Subtasks\Primer;
return [
// Task strategies
//
// Here you can configure in a modular way which tasks to use to
// execute various core parts of your deployment's flow
//////////////////////////////////////////////////////////////////////
// Which strategy to use to check the server
'check' => 'Php',
// Which strategy to use to create a new release
'deploy' => 'Clone',
// Which strategy to use to test your application
'test' => 'Phpunit',
// Which strategy to use to migrate your database
'migrate' => 'Artisan',
// Which strategy to use to install your application's dependencies
'dependencies' => 'Polyglot',
// Execution hooks
//////////////////////////////////////////////////////////////////////
'composer' => [
'install' => function (Composer $composer, $task) {
return $composer->install(
[],
[
'--no-interaction' => null,
'--no-dev' => null,
'--prefer-dist' => null,
'--optimize-autoloader' => null
]
);
},
'update' => function (Composer $composer) {
return $composer->update(
[],
[
'--optimize-autoloader' => null
]
);
},
],
// Here you can configure the Primer tasks
// which will run a set of commands on the local
// machine, determining whether the deploy can proceed
// or not
'primer' => function (Primer $task) {
return [
// $task->executeTask('Test'),
// $task->binary('grunt')->execute('lint'),
];
},
];

Note: This is a project readme, available because of the useful notes, some parts may not apply

Upon downloading the package, you will need to run composer install to install the vendor files. This also means that if you update or add a composer dependency eg (composer require or composer update) that you will need to let others know to rerun composer install on their personal machines.

Note: The deploy scripts take care of running composer during the deploy process.

Database Setup

    CREATE DATABASE resources COLLATE utf8_general_ci;
    CREATE USER 'user'@'%' IDENTIFIED BY 'password';
    GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER ON `resources`.* TO 'user'@'%';
    
    mysql root: password  
    replicator: password  

Setup EC2

  1. Install LAMP http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/install-LAMP.html

Master Elastic IP: note your ip

  1. Setup MySQL Master/Slave https://www.digitalocean.com/community/tutorials/how-to-set-up-mysql-master-master-replication

     # Master Setup        
     CREATE DATABASE resources COLLATE utf8_general_ci;
     CREATE USER 'user'@'%' IDENTIFIED BY 'password';
     GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, INDEX, ALTER ON `resources`.* TO 'user'@'%';
    
     # Slave Setup (first copy full MySQL from Master)
     CHANGE MASTER TO
         MASTER_HOST='masterInternalIp', // you have to make sure the master has this open for the slaves IP in EC2 security groups, in my tests setting it as open to the group name didn't work, it had to be the slave's IP
         MASTER_USER='repUser',
         MASTER_PASSWORD='repPassword',
         MASTER_LOG_FILE = 'mysql-bin.000003', MASTER_LOG_POS = 107;  // You'll grab this information from the master server, see the digital ocean post for more details
     REVOKE ALL PRIVILEGES ON `resources`.* FROM 'user'@'%'; // makes sure the resources user can't write to the slave since that wouldn't replicate back to the master
     GRANT SELECT ON `resources`.* TO 'user'@'%';
    

In my.cnf set expire_logs_days=30, this cleans out the binary logs.

Setting up an EC2 Server:

  1. Install Composer cd ~
    sudo curl -sS https://getcomposer.org/installer | sudo php
    sudo mv composer.phar /usr/local/bin/composer
    sudo ln -s /usr/local/bin/composer /usr/bin/composer
  2. Install Git, Mcrypt for Rocketeer, and MbString sudo yum install git php56-mcrypt php56-mbstring
  3. Update httpd.conf to be the local copy (You'll need to make the file global writable temporarily.)
    (You must get the pem file from amazon when you set up the server.) scp -i ~/.ssh/resources.pem httpd.conf ec2-user@serverUrl:/etc/httpd/conf/httpd.conf
    scp -i ~/.ssh/resources.pem httpd.conf ec2-user@serverURl:/etc/httpd/conf/httpd.conf
  4. Add apache to the www group
  5. Restart Apache sudo service httpd restart
  6. Laravel shared .env file vi /var/www/resources/shared/.env (Insert the needed variables using your local env/.env.example or the .env file from one of the other EC2 instances as a template.) ln -s /var/www/resources/shared/.env /var/www/resources/releases/20150707105432/.env

Adding a Server

  1. Create a new server
  2. Put it in the resources (You can name this whatever you want in your setup.) security group, resources IAM, use the Server2 Custom AMI. (I created the AMI from the installed slave server, you could do the same to make adding a new server easier.)
  3. On boot you may need to restore the db from a master backup and reconfigure the slave to the correct log instance. To avoid this, make a new AMI off of the slave instance just before creating the instance rather than using the older AMI.
  4. Add the servers public IP to rocketeer config with 'db_role' => false and add it to the 'default' array (line 33) and deploy
  5. You may wish to edit the server_name in the .env file.
    1. After this, you need to run php /var/www/resources/current/artisan config:cache
  6. Add the server to the load balancer

It should now be ready to go.

Master Failure/Reboot

  • If the master server ever fails, it can be restored from backups.

  • Whether it is just rebooted or fails, it's private IP will change which will result in the slave nodes being unable to connect. You will need to get into mysql shell and run show slave status; to ge the current master_log_file and Read_Master_Log_Pos. Then, use the change master syntax to set the new IP, log_file, and log_pos.

    CHANGE MASTER TO
        MASTER_HOST='masterInternalIp',
        MASTER_USER='repUser',
        MASTER_PASSWORD='repPassword',
        MASTER_LOG_FILE = 'mysql-bin.000003', MASTER_LOG_POS = 107; 
    REVOKE ALL PRIVILEGES ON `resources`.* FROM 'repUser'@'%';
    GRANT SELECT ON `resources`.* TO 'repUser'@'%';
    

Accessing the Servers

SSH

Master: `ssh -i ~/.ssh/resources.pem ec2-user@masterServerUrl`

Slave: ssh -i ~/.ssh/resources.pem ec2-user@slaveServerUrl

Deploy

Install Rocketeer Locally

$ wget http://rocketeer.autopergamene.eu/versions/rocketeer.phar
$ chmod +x rocketeer.phar
$ mv rocketeer.phar /usr/local/bin/rocketeer

To Deploy

rocketeer deploy --migrate

Other Notes

  • Don't use "shift31/laravel-elasticsearch": "~1.3", Sadly this uses the ES PHP API which uses Guzzle 3 which doesn't support the right HTTPS for Bonsai so it all doesn't work... (Thus, we implement our own with Guzzle 6.)
#
# This is the main Apache HTTP server configuration file. It contains the
# configuration directives that give the server its instructions.
# See <URL:http://httpd.apache.org/docs/2.4/> for detailed information.
# In particular, see
# <URL:http://httpd.apache.org/docs/2.4/mod/directives.html>
# for a discussion of each configuration directive.
#
# Do NOT simply read the instructions in here without understanding
# what they do. They're here only as hints or reminders. If you are unsure
# consult the online docs. You have been warned.
#
# Configuration and logfile names: If the filenames you specify for many
# of the server's control files begin with "/" (or "drive:/" for Win32), the
# server will use that explicit path. If the filenames do *not* begin
# with "/", the value of ServerRoot is prepended -- so 'log/access_log'
# with ServerRoot set to '/www' will be interpreted by the
# server as '/www/log/access_log', where as '/log/access_log' will be
# interpreted as '/log/access_log'.
#
# ServerRoot: The top of the directory tree under which the server's
# configuration, error, and log files are kept.
#
# Do not add a slash at the end of the directory path. If you point
# ServerRoot at a non-local disk, be sure to specify a local disk on the
# Mutex directive, if file-based mutexes are used. If you wish to share the
# same ServerRoot for multiple httpd daemons, you will need to change at
# least PidFile.
#
ServerRoot "/etc/httpd"
#
# Listen: Allows you to bind Apache to specific IP addresses and/or
# ports, instead of the default. See also the <VirtualHost>
# directive.
#
# Change this to Listen on specific IP addresses as shown below to
# prevent Apache from glomming onto all bound IP addresses.
#
#Listen 12.34.56.78:80
Listen 80
#
# Dynamic Shared Object (DSO) Support
#
# To be able to use the functionality of a module which was built as a DSO you
# have to place corresponding `LoadModule' lines at this location so the
# directives contained in it are actually available _before_ they are used.
# Statically compiled modules (those listed by `httpd -l') do not need
# to be loaded here.
#
# Example:
# LoadModule foo_module modules/mod_foo.so
#
Include conf.modules.d/*.conf
#
# If you wish httpd to run as a different user or group, you must run
# httpd as root initially and it will switch.
#
# User/Group: The name (or #number) of the user/group to run httpd as.
# It is usually good practice to create a dedicated user and group for
# running httpd, as with most system services.
#
User apache
Group apache
# 'Main' server configuration
#
# The directives in this section set up the values used by the 'main'
# server, which responds to any requests that aren't handled by a
# <VirtualHost> definition. These values also provide defaults for
# any <VirtualHost> containers you may define later in the file.
#
# All of these directives may appear inside <VirtualHost> containers,
# in which case these default settings will be overridden for the
# virtual host being defined.
#
#
# ServerAdmin: Your address, where problems with the server should be
# e-mailed. This address appears on some server-generated pages, such
# as error documents. e.g. [email protected]
#
ServerAdmin root@localhost
#
# ServerName gives the name and port that the server uses to identify itself.
# This can often be determined automatically, but we recommend you specify
# it explicitly to prevent problems during startup.
#
# If your host doesn't have a registered DNS name, enter its IP address here.
#
#ServerName www.example.com:80
#
# Deny access to the entirety of your server's filesystem. You must
# explicitly permit access to web content directories in other
# <Directory> blocks below.
#
<Directory />
AllowOverride none
Require all denied
</Directory>
#
# Note that from this point forward you must specifically allow
# particular features to be enabled - so if something's not working as
# you might expect, make sure that you have specifically enabled it
# below.
#
#
# DocumentRoot: The directory out of which you will serve your
# documents. By default, all requests are taken from this directory, but
# symbolic links and aliases may be used to point to other locations.
#
DocumentRoot "/var/www/directory/current/public"
#
# Relax access to content within /var/www.
#
#<Directory "/var/www/directory">
# AllowOverride All
# # Allow open access:
# Require all granted
#</Directory>
# Further relax access to the default document root:
<Directory "/var/www/directory/current/public">
#
# Possible values for the Options directive are "None", "All",
# or any combination of:
# Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews
#
# Note that "MultiViews" must be named *explicitly* --- "Options All"
# doesn't give it to you.
#
# The Options directive is both complicated and important. Please see
# http://httpd.apache.org/docs/2.4/mod/core.html#options
# for more information.
#
Options Indexes FollowSymLinks
#
# AllowOverride controls what directives may be placed in .htaccess files.
# It can be "All", "None", or any combination of the keywords:
# Options FileInfo AuthConfig Limit
#
AllowOverride All
#
# Controls who can get stuff from this server.
#
Require all granted
</Directory>
#
# DirectoryIndex: sets the file that Apache will serve if a directory
# is requested.
#
<IfModule dir_module>
DirectoryIndex index.php index.html
</IfModule>
#
# The following lines prevent .htaccess and .htpasswd files from being
# viewed by Web clients.
#
<Files ".ht*">
Require all denied
</Files>
#
# ErrorLog: The location of the error log file.
# If you do not specify an ErrorLog directive within a <VirtualHost>
# container, error messages relating to that virtual host will be
# logged here. If you *do* define an error logfile for a <VirtualHost>
# container, that host's errors will be logged there and not here.
#
ErrorLog "logs/error_log"
#
# LogLevel: Control the number of messages logged to the error_log.
# Possible values include: debug, info, notice, warn, error, crit,
# alert, emerg.
#
LogLevel warn
<IfModule log_config_module>
#
# The following directives define some format nicknames for use with
# a CustomLog directive (see below).
#
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
<IfModule logio_module>
# You need to enable mod_logio.c to use %I and %O
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
</IfModule>
#
# The location and format of the access logfile (Common Logfile Format).
# If you do not define any access logfiles within a <VirtualHost>
# container, they will be logged here. Contrariwise, if you *do*
# define per-<VirtualHost> access logfiles, transactions will be
# logged therein and *not* in this file.
#
#CustomLog "logs/access_log" common
#
# If you prefer a logfile with access, agent, and referer information
# (Combined Logfile Format) you can use the following directive.
#
CustomLog "logs/access_log" combined
</IfModule>
<IfModule alias_module>
#
# Redirect: Allows you to tell clients about documents that used to
# exist in your server's namespace, but do not anymore. The client
# will make a new request for the document at its new location.
# Example:
# Redirect permanent /foo http://www.example.com/bar
#
# Alias: Maps web paths into filesystem paths and is used to
# access content that does not live under the DocumentRoot.
# Example:
# Alias /webpath /full/filesystem/path
#
# If you include a trailing / on /webpath then the server will
# require it to be present in the URL. You will also likely
# need to provide a <Directory> section to allow access to
# the filesystem path.
#
# ScriptAlias: This controls which directories contain server scripts.
# ScriptAliases are essentially the same as Aliases, except that
# documents in the target directory are treated as applications and
# run by the server when requested rather than as documents sent to the
# client. The same rules about trailing "/" apply to ScriptAlias
# directives as to Alias.
#
#ScriptAlias /cgi-bin/ "/var/www/cgi-bin/"
</IfModule>
#
# "/var/www/cgi-bin" should be changed to whatever your ScriptAliased
# CGI directory exists, if you have that configured.
#
#<Directory "/var/www/cgi-bin">
# AllowOverride None
# Options None
# Require all granted
#</Directory>
<IfModule mime_module>
#
# TypesConfig points to the file containing the list of mappings from
# filename extension to MIME-type.
#
TypesConfig /etc/mime.types
#
# AddType allows you to add to or override the MIME configuration
# file specified in TypesConfig for specific file types.
#
#AddType application/x-gzip .tgz
#
# AddEncoding allows you to have certain browsers uncompress
# information on the fly. Note: Not all browsers support this.
#
#AddEncoding x-compress .Z
#AddEncoding x-gzip .gz .tgz
#
# If the AddEncoding directives above are commented-out, then you
# probably should define those extensions to indicate media types:
#
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
#
# AddHandler allows you to map certain file extensions to "handlers":
# actions unrelated to filetype. These can be either built into the server
# or added with the Action directive (see below)
#
# To use CGI scripts outside of ScriptAliased directories:
# (You will also need to add "ExecCGI" to the "Options" directive.)
#
#AddHandler cgi-script .cgi
# For type maps (negotiated resources):
#AddHandler type-map var
#
# Filters allow you to process content before it is sent to the client.
#
# To parse .shtml files for server-side includes (SSI):
# (You will also need to add "Includes" to the "Options" directive.)
#
AddType text/html .shtml
AddOutputFilter INCLUDES .shtml
</IfModule>
#
# Specify a default charset for all content served; this enables
# interpretation of all content as UTF-8 by default. To use the
# default browser choice (ISO-8859-1), or to allow the META tags
# in HTML content to override this choice, comment out this
# directive:
#
AddDefaultCharset UTF-8
<IfModule mime_magic_module>
#
# The mod_mime_magic module allows the server to use various hints from the
# contents of the file itself to determine its type. The MIMEMagicFile
# directive tells the module where the hint definitions are located.
#
MIMEMagicFile conf/magic
</IfModule>
#
# Customizable error responses come in three flavors:
# 1) plain text 2) local redirects 3) external redirects
#
# Some examples:
#ErrorDocument 500 "The server made a boo boo."
#ErrorDocument 404 /missing.html
#ErrorDocument 404 "/cgi-bin/missing_handler.pl"
#ErrorDocument 402 http://www.example.com/subscription_info.html
#
#
# EnableMMAP and EnableSendfile: On systems that support it,
# memory-mapping or the sendfile syscall may be used to deliver
# files. This usually improves server performance, but must
# be turned off when serving from networked-mounted
# filesystems or if support for these functions is otherwise
# broken on your system.
# Defaults if commented: EnableMMAP On, EnableSendfile Off
#
#EnableMMAP off
EnableSendfile on
# Supplemental configuration
#
# Load config files in the "/etc/httpd/conf.d" directory, if any.
IncludeOptional conf.d/*.conf
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment