|
#!/bin/bash |
|
|
|
: ' |
|
This script monitors Docker services for restart events and applies a temporary |
|
DNS fix by updating the /etc/hosts file within each container. Initially, the |
|
script performs the DNS fix for all relevant services, then continuously |
|
listens for any service restarts, waits for a brief period to ensure service |
|
stabilization, and reapplies the DNS fix as needed. It is designed for |
|
environments where container IP addresses must be consistently mapped in |
|
/etc/hosts across multiple services. Specifically, it is intended for ISLE-DC |
|
troubleshooting when containers have issues resolving, such as when behind a |
|
reverse proxy or on a cloud platform. |
|
|
|
=============================================================================== |
|
WARNING: Potential Issues with This Script |
|
------------------------------------------------------------------------------- |
|
|
|
This script is designed to monitor Docker service restart events and apply DNS |
|
fixes by modifying the /etc/hosts file within Docker containers. However, there |
|
are several inherent risks and drawbacks associated with this approach and is |
|
bad practice: |
|
|
|
1. **Performance Impact**: |
|
- The script continuously monitors Docker events, which can consume CPU |
|
resources even when no events are happening. This can impact the |
|
performance of both the host system and the running services, especially |
|
in environments with frequent service restarts or a large number of |
|
services. |
|
|
|
2. **Concurrency Issues**: |
|
- The script could be triggered multiple times concurrently if multiple |
|
services restart simultaneously or in quick succession. This can lead to |
|
race conditions, potentially causing inconsistent /etc/hosts entries. |
|
|
|
3. **Security Risks**: |
|
- Modifying /etc/hosts inside containers with elevated privileges could |
|
expose the system to security vulnerabilities, particularly if the script |
|
is altered or misused. |
|
|
|
4. **Maintenance Complexity**: |
|
- The script introduces additional complexity and maintenance overhead. As |
|
the environment evolves, keeping the script up-to-date with all the |
|
necessary service configurations could become cumbersome. |
|
|
|
5. **Lack of Robust Error Handling**: |
|
- The script does not have comprehensive error handling or logging. If an |
|
error occurs (e.g., a container fails to restart, or the script cannot |
|
update /etc/hosts), the system may end up in an inconsistent state, and |
|
diagnosing the issue could be difficult. |
|
|
|
6. **Potential for Overriding Configurations**: |
|
- Since the script adds to /etc/hosts based on specific logic, it will |
|
overwhelm /etc/hosts with changes made by other processes or scripts, |
|
potentially leading to a long hosts file. |
|
|
|
7. **Scalability Concerns**: |
|
- As the number of services grows, the script may become less efficient and |
|
harder to manage, making it unsuitable for larger or more complex |
|
environments. |
|
|
|
------------------------------------------------------------------------------- |
|
ALTERNATIVE RECOMMENDATIONS: |
|
------------------------------------------------------------------------------- |
|
|
|
- **Resolve the Issue causing this**: This is intended as a quick fix and to |
|
bring to light a bigger issue within docker compose with isle-dc. |
|
|
|
- **Implement Health Checks**: Use Docker’s health check functionality combined |
|
with proper service dependencies and startup logic to ensure services start |
|
in the correct order. |
|
|
|
=============================================================================== |
|
' |
|
|
|
# Constants for temp DNS fix |
|
SERVICE_ALIASES=("crayfits" "homarus" "houdini" "hypercube" "cantaloupe" "drupal" "milliner" "alpaca" "crayfish" "activemq" "fits") |
|
COPY_SERVICE_ALIASES=("${SERVICE_ALIASES[@]}") |
|
|
|
# Function to retrieve the first IP address of the target container. |
|
get_first_container_ip() { |
|
local container_name="$1" |
|
docker inspect -f '{{range $k, $v := .NetworkSettings.Networks}}{{printf "%s\n" $v.IPAddress}}{{end}}' "$container_name" | head -n 1 |
|
} |
|
|
|
# Function to check existing entries in /etc/hosts. |
|
check_existing_entries() { |
|
local dest_container="$1" |
|
local hostname="$2" |
|
docker exec "$dest_container" sh -c "grep -E '^[0-9\.]+\s+${hostname}$' /etc/hosts" |
|
} |
|
|
|
# Function to perform the temp DNS fix |
|
temp_dns_fix() { |
|
for CONTAINER_TO_ADD_THIS_TO in "${COPY_SERVICE_ALIASES[@]}"; do |
|
for SERVICE_ALIAS in "${SERVICE_ALIASES[@]}"; do |
|
if [[ "${SERVICE_ALIAS}" == "crayfish" ]] || [[ "${CONTAINER_TO_ADD_THIS_TO}" == "crayfish" ]]; then |
|
continue; |
|
fi |
|
if [[ "${SERVICE_ALIAS}" == "${CONTAINER_TO_ADD_THIS_TO}" ]]; then |
|
continue; |
|
fi |
|
|
|
TARGET_CONTAINER_NAME="isle-dc-${SERVICE_ALIAS}-1" |
|
DESTINATION_CONTAINER_NAME="isle-dc-${CONTAINER_TO_ADD_THIS_TO}-1" |
|
HOSTNAME_TO_UPDATE="$SERVICE_ALIAS" |
|
# Retrieve the first IP address of the target container. |
|
local target_container_ip |
|
target_container_ip=$(get_first_container_ip "$TARGET_CONTAINER_NAME") |
|
|
|
# Check if the IP address was retrieved successfully. |
|
if [[ -z "$target_container_ip" ]]; then |
|
printf "Error: Unable to retrieve IP address for container %s\n" "$TARGET_CONTAINER_NAME" >&2 |
|
continue |
|
fi |
|
|
|
# Specific case for cantaloupe container when processing drupal |
|
if [[ "$SERVICE_ALIAS" == "drupal" ]] && [[ "$CONTAINER_TO_ADD_THIS_TO" == "cantaloupe" ]]; then |
|
HOSTNAME_TO_UPDATE="idc-staging.cloud.library.jhu.edu" |
|
fi |
|
|
|
local existing_entries |
|
existing_entries=$(check_existing_entries "$DESTINATION_CONTAINER_NAME" "$HOSTNAME_TO_UPDATE") |
|
|
|
local match_found=false |
|
local conflict_found=false |
|
|
|
while IFS= read -r entry; do |
|
local existing_ip |
|
existing_ip=$(printf "%s" "$entry" | awk '{print $1}') |
|
|
|
if [[ "$existing_ip" == "$target_container_ip" ]]; then |
|
match_found=true |
|
else |
|
conflict_found=true |
|
fi |
|
done <<< "$existing_entries" |
|
|
|
if [[ "$conflict_found" == true ]]; then |
|
printf "Warning: Multiple IPs assigned to %s in /etc/hosts of %s. Manual editing required.\n" "$HOSTNAME_TO_UPDATE" "$DESTINATION_CONTAINER_NAME" >&2 |
|
fi |
|
|
|
if [[ "$match_found" == true ]]; then |
|
printf "Entry %s %s already exists with the correct IP in /etc/hosts of %s\n" "$target_container_ip" "$HOSTNAME_TO_UPDATE" "$DESTINATION_CONTAINER_NAME" |
|
elif [[ "$conflict_found" == true ]]; then |
|
docker exec "$DESTINATION_CONTAINER_NAME" sh -c "printf '%s\t%s\n' '$target_container_ip' '$HOSTNAME_TO_UPDATE' >> /etc/hosts" |
|
else |
|
docker exec "$DESTINATION_CONTAINER_NAME" sh -c "printf '%s\t%s\n' '$target_container_ip' '$HOSTNAME_TO_UPDATE' >> /etc/hosts" |
|
printf "Added %s %s to /etc/hosts of %s\n" "$target_container_ip" "$HOSTNAME_TO_UPDATE" "$DESTINATION_CONTAINER_NAME" |
|
fi |
|
printf "\nContents of %s : /etc/hosts \n---\n" "$DESTINATION_CONTAINER_NAME" |
|
docker exec "$DESTINATION_CONTAINER_NAME" sh -c "cat /etc/hosts" |
|
done |
|
done |
|
} |
|
|
|
# Function to check if the service is in the monitored array |
|
is_monitored_service() { |
|
local service_name="$1" |
|
for service in "${SERVICE_ALIASES[@]}"; do |
|
if [[ "$service_name" == *"$service"* ]]; then |
|
return 0 |
|
fi |
|
done |
|
return 1 |
|
} |
|
|
|
# Run the temp DNS fix initially |
|
echo "Running initial DNS fix..." |
|
temp_dns_fix |
|
|
|
# Start watching for service restarts |
|
echo -e "\nDone.\n\nWatching All isle-dc services for restarts: \n" |
|
docker events --filter 'event=start' --filter 'event=restart' | |
|
while read event |
|
do |
|
# Extract the service name from the event (usually the last part of the string) |
|
service_name=$(echo "$event" | awk '{print $NF}') |
|
# Check if the service name is in the monitored array |
|
if is_monitored_service "$service_name"; then |
|
echo "Service $service_name started or restarted, waiting for 10 seconds to ensure stabilization..." |
|
sleep 10 # Wait for 10 seconds before running the DNS fix again |
|
temp_dns_fix |
|
echo -e "\Done.\n\nWatching All isle-dc services for restarts: \n" |
|
fi |
|
done |
