Skip to content

Instantly share code, notes, and snippets.

@WMAL

WMAL/lsyncd_setup.sh

Last active February 20, 2025 09:30
Show Gist options
  • Save WMAL/dc1139f7df1770a584de6296f0447845 to your computer and use it in GitHub Desktop.
Save WMAL/dc1139f7df1770a584de6296f0447845 to your computer and use it in GitHub Desktop.
Download ZIP
Lsyncd (Live Syncing Daemon) is a powerful tool for keeping files synchronized across multiple servers in real time. Whether managing multiple worker nodes or ensuring a master node is always up to date, Lsyncd provides an efficient, automated solution. This guide walks you through setting up Lsyncd on a Debian-based system using a Bash script.
Raw
lsyncd_setup.sh
#!/bin/bash
# =========================================
# Lsyncd Setup Script
# =========================================
#
# Version: 1.0.1
# Script written by Warith AL Maawali
#
# Discord channel: https://discord.gg/KEFErEx
# Twitter: http://twitter.com/warith2020
# Linkedin: http://www.linkedin.com/in/warith1977
# Website: https://www.digi77.com
# (c) 2025
#
# Description:
# This script sets up Lsyncd (Live Syncing Daemon) on a Debian-based system.
# It configures Lsyncd to synchronize content from worker nodes to a master node.
# Sets up a dedicated user for syncing, and configures SSH for secure communication.
#
# For more information, visit: https://github.com/axkibe/lsyncd
#
# This software is dual-licensed:
#
# Personal, non-commercial use: Apache License 2.0
# Commercial, corporate, or organizational use: Separate commercial license required.
# Contact me for licensing inquiries.
#
# Usage: ./lsyncd_setup.sh
#
# Usage Examples:
# Run this script as root to set up Lsyncd:
# ./lsyncd_setup.sh
# =========================================
# Required packages
packages=(sudo lsyncd rsync)
# Check if SSH server is running, if not add it to required packages
if ! systemctl is-active --quiet sshd; then
packages+=(openssh-server)
fi
# Check if dnsutils is installed, if not add it to required packages
if ! command -v dig &>/dev/null; then
packages+=(dnsutils)
fi
extra_packages=()
# Lsyncd variables
SYNC_USER="syncuser"
MASTER_SYNC_DIR="/var/www/html/cards"
WORKER_SYNC_DIR="/root/cards"
SSH_PORT="22222"
LSYNCD_CONF="/etc/lsyncd/lsyncd.conf.lua"
SSH_CONFIG="/etc/ssh/sshd_config.d/lsyncd_syncuser.conf"
# Master node (can be IP or domain)
MASTER_NODE="example.com" # Change this to your master node IP or domain
# Resolve master node to IP if it's a domain
MASTER_IP=$(dig +short "$MASTER_NODE" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -n1)
if [ -n "$MASTER_IP" ]; then
MASTER_NODE_IP="$MASTER_IP"
else
MASTER_NODE_IP="$MASTER_NODE"
fi
# Worker nodes that will sync to master (can be IPs or domains)
WORKER_NODES=(
"34.218.156.89"
"52.142.93.176"
"104.198.45.123"
)
# SSH keys for lsyncd
PRIVATE_KEY=$(
cat <<'EOL'
-----BEGIN OPENSSH PRIVATE KEY-----
<YOUR_PRIVATE_KEY_HERE>
-----END OPENSSH PRIVATE KEY-----
EOL
)
# Public key for lsyncd
PUBLIC_KEY="<YOUR_PUBLIC_KEY_HERE>"
# Function to install required packages
install_required_packages() {
echo "Installing required packages..."
for package in "${packages[@]}"; do
if ! command -v "$package" &>/dev/null; then
echo "Installing $package..."
sudo apt update
sudo apt install -y "$package"
else
echo "$package is already installed."
fi
done
}
# Function to set up Lsyncd
setup_lsyncd() {
# Step 1: Install required packages
install_required_packages
# Step 2: Create a dedicated user
echo "Creating dedicated user '$SYNC_USER'..."
sudo adduser --disabled-password --gecos "" $SYNC_USER
# Step 3: Set up directory and permissions
# Get current node's IP
CURRENT_IP=$(hostname -I | awk '{print $1}')
# Determine if this is master or worker node
IS_MASTER=false
if [ "$CURRENT_IP" = "$MASTER_NODE_IP" ]; then
IS_MASTER=true
SYNC_DIR="$MASTER_SYNC_DIR"
else
SYNC_DIR="$WORKER_SYNC_DIR"
fi
echo "Setting up permissions on $SYNC_DIR..."
sudo mkdir -p $SYNC_DIR
sudo chown -R $SYNC_USER:$SYNC_USER $SYNC_DIR
# Step 4: Create lsyncd configuration file
echo "Creating lsyncd configuration file..."
rm -f $LSYNCD_CONF
# Ensure the directory exists before creating the file
mkdir -p $(dirname $LSYNCD_CONF)
# Create log directory and file, set permissions
sudo mkdir -p /var/log/lsyncd
sudo touch /var/log/lsyncd.log
sudo chmod 666 /var/log/lsyncd.log
sudo touch /var/run/lsyncd.status
sudo chmod 666 /var/run/lsyncd.status
# Write Lsyncd configuration based on node type
if $IS_MASTER; then
# Master node configuration
bash -c "cat > $LSYNCD_CONF" <<EOL
settings {
logfile = "/var/log/lsyncd.log",
statusFile = "/var/run/lsyncd.status",
inotifyMode = "CloseWrite",
}
sync {
default.rsync,
source = "/var/www/html/cards",
target = "/var/www/html/cards",
delay = 15,
}
EOL
else
# Worker node configuration
bash -c "cat > $LSYNCD_CONF" <<EOL
settings {
logfile = "/var/log/lsyncd.log",
statusFile = "/var/run/lsyncd.status",
inotifyMode = "CloseWrite",
}
sync {
default.rsync,
source = "$WORKER_SYNC_DIR",
target = "$SYNC_USER@$MASTER_NODE:$MASTER_SYNC_DIR",
delay = 15,
rsync = {
archive = true,
compress = true,
_extra = {"-e", "ssh -p $SSH_PORT -o StrictHostKeyChecking=no -i /home/$SYNC_USER/.ssh/id_ed25519"},
},
}
EOL
fi
rm -f $SSH_CONFIG
# Step 5: Configure SSH
echo "Configuring SSH for $SYNC_USER..."
# Set the SSH Port outside of the Match block
echo "Port $SSH_PORT" | sudo tee -a $SSH_CONFIG
# Create SSH configuration for the sync user with allowed IPs/domains
echo "Match User $SYNC_USER" | sudo tee -a $SSH_CONFIG
echo " PasswordAuthentication no" | sudo tee -a $SSH_CONFIG
echo -n " AllowUsers" | sudo tee -a $SSH_CONFIG
# Add master node to allowed users
echo -n " $SYNC_USER@$MASTER_NODE_IP" | sudo tee -a $SSH_CONFIG
# Add all worker nodes to allowed users
for node in "${WORKER_NODES[@]}"; do
echo -n " $SYNC_USER@$node" | sudo tee -a $SSH_CONFIG
done
echo "" | sudo tee -a $SSH_CONFIG
# Step 6: Set up SSH keys for syncuser
echo "Setting up SSH keys for $SYNC_USER..."
sudo -u $SYNC_USER mkdir -p /home/$SYNC_USER/.ssh
if [ ! -f /home/$SYNC_USER/.ssh/id_ed25519 ]; then
sudo -u $SYNC_USER bash -c "echo '$PRIVATE_KEY' > /home/$SYNC_USER/.ssh/id_ed25519"
fi
if [ ! -f /home/$SYNC_USER/.ssh/id_ed25519.pub ]; then
sudo -u $SYNC_USER bash -c "echo '$PUBLIC_KEY' > /home/$SYNC_USER/.ssh/id_ed25519.pub"
fi
if ! grep -q "$PUBLIC_KEY" /home/$SYNC_USER/.ssh/authorized_keys; then
sudo -u $SYNC_USER bash -c "cat /home/$SYNC_USER/.ssh/id_ed25519.pub >> /home/$SYNC_USER/.ssh/authorized_keys"
fi
sudo chmod 600 /home/$SYNC_USER/.ssh/id_ed25519
sudo chmod 644 /home/$SYNC_USER/.ssh/id_ed25519.pub
sudo chmod 600 /home/$SYNC_USER/.ssh/authorized_keys
sudo chown -R $SYNC_USER:$SYNC_USER /home/$SYNC_USER/.ssh
# Restart SSH service and handle potential errors
echo "Restarting SSH service..."
if ! sudo systemctl restart sshd; then
echo "Job for ssh.service failed because the control process exited with error code."
echo "See 'systemctl status ssh.service' and 'journalctl -xeu ssh.service' for details."
exit 1
fi
sudo systemctl restart sshd
# Step 7: Start and enable lsyncd
echo "Starting lsyncd service..."
sudo systemctl start lsyncd
sudo systemctl enable lsyncd
if $IS_MASTER; then
echo -e "Setup complete. This is the master node that will receive content from worker nodes in $MASTER_SYNC_DIR."
else
echo -e "Setup complete. This worker node will sync its $WORKER_SYNC_DIR to master node $MASTER_NODE:$MASTER_SYNC_DIR."
fi
}
# Check for root privileges
if [ "$EUID" -ne 0 ]; then
echo "ERROR: This script must be run as root"
exit 1
fi
# Run the installation if script is executed directly
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
setup_lsyncd
fi
@WMAL
Copy link
Author

WMAL commented Feb 20, 2025

Alternative Approach – Mesh Synchronization:

Unlike traditional master-worker synchronization setups, another approach is a mesh topology where every node syncs with every other node. The alternative script provided here implements such a setup. Instead of synchronizing only from worker nodes to a single master node, this script configures Lsyncd to synchronize files bidirectionally across multiple nodes. This ensures that all nodes maintain identical data without relying on a central master server. The script dynamically generates lsyncd.conf.lua configurations for each participating node and uses SSH authentication to securely exchange data. This setup is particularly useful for distributed environments where redundancy and failover capabilities are critical.


#!/bin/bash
# =========================================
# Lsyncd Setup Script
# =========================================
#
# Version: 1.0.0
# Script written by Warith AL Maawali
#
# Discord channel: https://discord.gg/KEFErEx
# Twitter: http://twitter.com/warith2020
# Linkedin: http://www.linkedin.com/in/warith1977
# Website: https://www.digi77.com
# (c) 2025
#
# Description:
# This script sets up Lsyncd (Live Syncing Daemon) on a Debian-based system.
# It configures Lsyncd to synchronize content between multiple nodes in a mesh topology.
# Sets up a dedicated user for syncing, and configures SSH for secure communication.
#
# For more information, visit: https://github.com/axkibe/lsyncd
#
# This software is dual-licensed:
#
# Personal, non-commercial use: Apache License 2.0
# Commercial, corporate, or organizational use: Separate commercial license required.
# Contact me for licensing inquiries.
#
# Usage: ./lsync-setup-all-same-all-sync.sh
#
# Usage Examples:
#   Run this script as root to set up Lsyncd:
#     ./lsync-setup-all-same-all-sync.sh
# =========================================

# Define variables for configuration
SYNC_USER="syncuser"                                      # Username for sync operations
SYNC_DIR="/var/www/html"                                 # Directory to synchronize
SSH_PORT="54028"                                         # Custom SSH port for security
LSYNCD_CONF="/etc/lsyncd/lsyncd.conf.lua"               # Lsyncd config file location
SSH_CONFIG="/etc/ssh/sshd_config.d/lsyncd_syncuser.conf" # SSH config for sync user

# IP addresses of nodes to sync (replace with your actual IPs)
NODE_IPS=("34.218.156.89" "52.142.93.176") 
NODE_IP1="34.218.156.89"
NODE_IP2="52.142.93.176"

# SSH keys for authentication
# Replace with your own generated keys
PRIVATE_KEY=$(
  cat <<'EOL'
-----BEGIN OPENSSH PRIVATE KEY-----
<YOUR_PRIVATE_KEY_HERE>
-----END OPENSSH PRIVATE KEY-----
EOL
)

PUBLIC_KEY="ssh-ed25519 AAAAB3NzaC1yc2EAAAADAQABAAABAQC... your_public_key"

setup_lsyncd() {
  # Step 1: Create a dedicated user for sync operations
  echo "Creating dedicated user '$SYNC_USER'..."
  sudo adduser --disabled-password --gecos "" $SYNC_USER

  # Step 2: Set up directory and permissions
  echo "Setting up permissions on $SYNC_DIR..."
  sudo mkdir -p $SYNC_DIR
  sudo chown -R $SYNC_USER:$SYNC_USER $SYNC_DIR

  # Step 3: Install lsyncd if not present
  echo "Checking if lsyncd is installed..."
  if ! command -v lsyncd &>/dev/null; then
    echo "lsyncd not found, installing..."
    sudo apt update
    sudo apt install -y lsyncd
  else
    echo "lsyncd is already installed."
  fi

  # Step 4: Create lsyncd configuration file
  echo "Creating lsyncd configuration file..."
  rm -f $LSYNCD_CONF
  # Ensure the directory exists before creating the file
  mkdir -p $(dirname $LSYNCD_CONF)

  # Create log directory and file, set permissions
  sudo mkdir -p /var/log/lsyncd
  sudo touch /var/log/lsyncd.log
  sudo chmod 666 /var/log/lsyncd.log
  sudo touch /var/run/lsyncd.status
  sudo chmod 666 /var/run/lsyncd.status

  # Write initial Lsyncd settings to the config file
  bash -c "cat > $LSYNCD_CONF" <<EOL
settings {
    logfile    = "/var/log/lsyncd.log",
    statusFile = "/var/run/lsyncd.status",
    inotifyMode = "Modify",    
  }
EOL

  # Append sync configurations for each target node except the current host
  for NODE in "${NODE_IPS[@]}"; do
    if [ "$NODE" != "$(hostname -I | awk '{print $1}')" ]; then
      bash -c "cat >> $LSYNCD_CONF" <<EOL
sync {
    default.rsync,
    source = "$SYNC_DIR",
    target = "$SYNC_USER@$NODE:$SYNC_DIR",
    rsync = {
        archive = true,
        compress = true,         
        _extra = {"--delete", "-e", "ssh -p $SSH_PORT -o StrictHostKeyChecking=no -i /home/$SYNC_USER/.ssh/id_ed25519"},
    },
}
EOL
    fi
  done

  rm -f $SSH_CONFIG
  # Step 5: Configure SSH for sync user
  echo "Configuring SSH for $SYNC_USER..."

  # Set the SSH Port outside of the Match block
  echo "Port $SSH_PORT" | sudo tee -a $SSH_CONFIG

  # Create SSH configuration for the sync user
  bash -c "cat >> $SSH_CONFIG" <<EOL
Match User $SYNC_USER
    PasswordAuthentication no
    AllowUsers $SYNC_USER@$NODE_IP1 $SYNC_USER@$NODE_IP2
EOL

  # Step 6: Set up SSH keys for syncuser
  echo "Setting up SSH keys for $SYNC_USER..."
  sudo -u $SYNC_USER mkdir -p /home/$SYNC_USER/.ssh
  if [ ! -f /home/$SYNC_USER/.ssh/id_ed25519 ]; then
    sudo -u $SYNC_USER bash -c "echo '$PRIVATE_KEY' > /home/$SYNC_USER/.ssh/id_ed25519"
  fi

  if [ ! -f /home/$SYNC_USER/.ssh/id_ed25519.pub ]; then
    sudo -u $SYNC_USER bash -c "echo '$PUBLIC_KEY' > /home/$SYNC_USER/.ssh/id_ed25519.pub"
  fi

  if ! grep -q "$PUBLIC_KEY" /home/$SYNC_USER/.ssh/authorized_keys; then
    sudo -u $SYNC_USER bash -c "cat /home/$SYNC_USER/.ssh/id_ed25519.pub >> /home/$SYNC_USER/.ssh/authorized_keys"
  fi
  sudo chmod 600 /home/$SYNC_USER/.ssh/id_ed25519
  sudo chmod 644 /home/$SYNC_USER/.ssh/id_ed25519.pub
  sudo chmod 600 /home/$SYNC_USER/.ssh/authorized_keys
  sudo chown -R $SYNC_USER:$SYNC_USER /home/$SYNC_USER/.ssh

  # Restart SSH service and handle potential errors
  echo "Restarting SSH service..."
  if ! sudo systemctl restart sshd; then
    echo "Job for ssh.service failed because the control process exited with error code."
    echo "See 'systemctl status ssh.service' and 'journalctl -xeu ssh.service' for details."
    exit 1
  fi

  sudo systemctl restart sshd

  # Step 7: Start and enable lsyncd service
  echo "Starting lsyncd service..."
  sudo systemctl start lsyncd
  sudo systemctl enable lsyncd

  echo "Setup complete. lsyncd is now monitoring $SYNC_DIR for synchronization with other nodes."
}

# Execute the setup function
setup_lsyncd

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment