Installation script for Mail Piler (open source version) with Manticore on Ubuntu 24.04 Noble (credit cmer)

piler.sh

#!/bin/bash
#
#
# This script installs the latest open source version of Mail Piler (mailpiler.org) from the master
# branch on GitHub by compiling it from source. It also installs all dependencies, including a MySQL database.
#
# You should run this script as root on a vanilla Ubuntu 24.04 installation.
#
#
set -o errexit
set -o pipefail
set -o nounset
set -x

PILER_HOSTNAME="${PILER_HOSTNAME:-piler.yourdomain.com}"
MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-yourpassword}"
MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-yourpassword}"
SERVER_ID="${SERVER_ID:-0}"
USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
PHP_FPM_SOCKET="/var/run/php/php8.3-fpm.sock"

MYSQL_HOSTNAME="localhost"
MYSQL_DATABASE="piler"
MYSQL_USERNAME="piler"

PILER_TARBALL="https://github.com/jsuto/piler/archive/refs/heads/master.zip"
PILER_USER="piler"
CONFIG_SITE_PHP="/etc/piler/config-site.php"
CONFIG_SITE_DIST_PHP="/etc/piler/config-site.dist.php"

export DEBIAN_FRONTEND=noninteractive

install_prerequisites() {
   apt-get update
   apt-get -y --no-install-recommends install \
      wget rsyslog openssl sysstat php8.3-cli php8.3-cgi php8.3-mysql php8.3-fpm php8.3-zip php8.3-ldap \
      php8.3-gd php8.3-curl php8.3-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip-dev \
      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client \
      mariadb-server build-essential libssl-dev libtre-dev libzip-dev libcurl4-openssl-dev unzip
}

install_manticore() {
	wget https://repo.manticoresearch.com/manticore-repo.noarch.deb
	dpkg -i manticore-repo.noarch.deb
	apt update
	apt -y install manticore manticore-extra
}	


create_user() {
   egrep -i "^x$PILER_USER:" /etc/passwd || adduser --no-create-home --disabled-password --disabled-login --gecos "" $PILER_USER
#   egrep -i "^x$PILER_USER:" /etc/passwd || adduser --disabled-password --gecos "" --shell /bin/bash $PILER_USER
}

create_mysql_user_and_database() {
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE USER IF NOT EXISTS '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "SET PASSWORD FOR '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' = PASSWORD('$MYSQL_PILER_PASSWORD');"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS $MYSQL_DATABASE CHARACTER SET 'utf8mb4';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "GRANT ALL PRIVILEGES ON $MYSQL_DATABASE.* to '$MYSQL_USERNAME'@'$MYSQL_HOSTNAME' IDENTIFIED BY '$MYSQL_PILER_PASSWORD';"
   mysql -u root -p"$MYSQL_ROOT_PASSWORD" -e "FLUSH PRIVILEGES;"

   mysql -u $MYSQL_USERNAME -p"$MYSQL_PILER_PASSWORD" $MYSQL_DATABASE < /etc/piler/db-mysql.sql
}

fix_mysql_settings() {
   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
[mysqld]
innodb_buffer_pool_size=512M
innodb_flush_log_at_trx_commit=1
innodb_log_buffer_size=64M
innodb_log_file_size=64M
innodb_read_io_threads=4
innodb_write_io_threads=4
innodb_log_files_in_group=2
innodb_file_per_table
PILER_CNF
}

start_mysql() {
   fix_mysql_settings
   service mysql restart
}

install_piler() {
   mkdir -p tmp && wget "${PILER_TARBALL}" -O tmp/piler-master.zip
   unzip tmp/piler-master -d tmp
   pushd tmp/piler-master
   ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb --enable-tcpwrappers --enable-memcached
   make clean all install
   cp ./contrib/webserver/piler-nginx.conf /etc/piler/piler-nginx.conf.dist
   cp ./etc/sphinx.conf.dist /etc/piler/sphinx.conf
   cp ./util/db-mysql.sql /etc/piler/db-mysql.sql
   popd

   crontab -u "$PILER_USER" /usr/share/piler/piler.cron

   touch /var/piler/.bash_history
   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
}

create_my_cnf() {
   local user=$1
   local password=$2
   local my_cnf=$3

   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
   chown $PILER_USER:$PILER_USER "$my_cnf"
   chmod 600 "$my_cnf"
}


fix_config_site_php() {
   cp $CONFIG_SITE_DIST_PHP $CONFIG_SITE_PHP
   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"

   {
      echo "\$config['SERVER_ID'] = $SERVER_ID;"
      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
      echo "\$config['SPHINX_VERSION'] = 331;"
   } >> "$CONFIG_SITE_PHP"

   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
   fi

   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
}


add_systemd_services() {
   pushd /etc/systemd/system

   ln -sf /usr/libexec/piler/pilersearch.service .
   ln -sf /usr/libexec/piler/piler.service .
   ln -sf /usr/libexec/piler/piler-smtp.service .

   popd

   systemctl daemon-reload

   systemctl enable pilersearch
   systemctl enable piler
   systemctl enable piler-smtp
}

create_cipher_key() {
   dd if=/dev/urandom bs=56 count=1 of=/etc/piler/piler.key
   chmod 640 /etc/piler/piler.key
   chown piler:piler /etc/piler/piler.key
}


fix_configs() {
	touch /etc/piler/MANTICORE
   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
      ln -s /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf

      nginx -t
      nginx -s reload
   fi

   if [[ ! -f /etc/piler/piler.conf ]]; then
      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
      chmod 600 /etc/piler/piler.conf
      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
   fi

	cp /etc/piler/manticore.conf.dist /etc/piler/manticore.conf
   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
          /etc/piler/manticore.conf
	
}

create_run_piler() {
cat <<EOF > /etc/systemd/system/pilerrun.service
[Unit]
Description=Create piler directory in /var/run
After=network.target

[Service]
Type=simple
ExecStartPre=-/usr/bin/mkdir /var/run/piler
ExecStart=/usr/bin/chown piler:piler /var/run/piler
Restart=on-abort

[Install]
WantedBy=multi-user.target
EOF

systemctl start pilerrun.service
systemctl enable pilerrun.service
}

install_prerequisites
install_manticore
create_user
install_piler
create_mysql_user_and_database
start_mysql

create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf

fix_configs
fix_config_site_php

add_systemd_services
create_cipher_key
create_run_piler

systemctl start pilersearch
systemctl start piler
systemctl start piler-smtp

your script works perfectly 👌

Glad it worked for you!

I recently installed it on a clean Ubuntu 24.04 server and it gave me an error because the unzip package was not installed. It would be good if you added the unzip package in the installation of packages and dependencies.

I recently installed it on a clean Ubuntu 24.04 server and it gave me an error because the unzip package was not installed. It would be good if you added the unzip package in the installation of packages and dependencies.

Done.

Hi. I am not so familiar with Docker install, but do you know which underlying OS was installed ? In my case when I installed on Debian it the webpage wasn't initially accessible.  But then I discovered that Debian 12 installs php8.2, but the script specified php8.3. I modified all  script's php entries to php8.2. Then on the Nginx config itself I also had to modify: fastcgi_pass unix:/run/php/php8.3-fpm.sock; to fastcgi_pass unix:/run/php/php8.2-fpm.sock; Regards, Clement On Tuesday, December 10, 2024, 12:58:17 AM GMT+3, Ryan Harris ***@***.***> wrote: @ShiftyMcCool commented on this gist. Hello! I ran the script and everything finished successfully and the services are running. However, when I attempt to hit the UI at port 80 in my browser, I get a generic nginx page. Am I doing something wrong? I've only ever run the docker version of Mail Piler so I'm guessing it's my fault :P Thanks! — Reply to this email directly, view it on GitHub or unsubscribe. You are receiving this email because you are subscribed to this thread. Triage notifications on the go with GitHub Mobile for iOS or Android.                                                            

Hi. I am not so familiar with Docker install, but do you know which underlying OS was installed ? In my case when I installed on Debian it the webpage wasn't initially accessible.  But then I discovered that Debian 12 installs php8.2, but the script specified php8.3. I modified all  script's php entries to php8.2. Then on the Nginx config itself I also had to modify: fastcgi_pass unix:/run/php/php8.3-fpm.sock; to fastcgi_pass unix:/run/php/php8.2-fpm.sock; Regards, Clement On Tuesday, December 10, 2024, 12:58:17 AM GMT+3, Ryan Harris @.***> wrote: @ShiftyMcCool commented on this gist. Hello! I ran the script and everything finished successfully and the services are running. However, when I attempt to hit the UI at port 80 in my browser, I get a generic nginx page. Am I doing something wrong? I've only ever run the docker version of Mail Piler so I'm guessing it's my fault :P Thanks! — Reply to this email directly, view it on GitHub or unsubscribe. You are receiving this email because you are subscribed to this thread. Triage notifications on the go with GitHub Mobile for iOS or Android.

For me the same problem.Only default nginx page is available.

Nice script, but package libcurl4-openssl-dev required by piler building process but not included in this script.

For those who trying to import mail from imap.yandex.com - you need to change regex in /usr/libexec/piler/imapfetch.py:58
Old str: f = re.split(r' \"[\/\.\\]+\" ', folder)
New str: f = re.split(r'\"\|\"\s*(\S+)', folder)

I Added libcurl4-openssl-dev to the install and checks for file existence before attempting operations so you can continue where left off , and completion message , how do you add the updated code?

I added libcurl4-openssl-dev to the prerequisites. I didn't have an issue on my installs, but can't hurt much to add it if some distribution needs it.