Upgrade Dataverse v6.0 to v6.1

upgrade_6_0_to_6_1.sh

#!/bin/bash

# Globals
DOMAIN="dataverse-clone.mse.jhu.edu"
PAYARA="/usr/local/payara"
DATAVERSE_WAR_URL="https://github.com/IQSS/dataverse/releases/download/v6.1/dataverse-6.1.war"
DATAVERSE_WAR_FILE="/home/dataverse/dataverse-6.1.war"
DATAVERSE_WAR_HASH="c6e931a7498c9d560782378d62b9444699d72b9c28f82f840ec4a4ba04b72771"
GEOSPATIAL_URL="https://github.com/IQSS/dataverse/releases/download/v6.1/geospatial.tsv"
GEOSPATIAL_FILE="/tmp/geospatial.tsv"
CITATION_URL=""https://github.com/IQSS/dataverse/releases/download/v6.1/citation.tsv
CITATION_FILE="/tmp/citation.tsv"
SOLR_FIELD_UPDATER_URL="https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.3.0/update-fields.sh"
SOLR_FIELD_UPDATER_FILE="/tmp/update-fields.sh"
DEPLOY_DIR="$PAYARA/glassfish/domains/domain1/generated"
CURRENT_VERSION="6.0"
TARGET_VERSION="6.1"
DATAVERSE_USER="dataverse"
BASHRC_FILE="/home/dataverse/.bashrc"
PAYARA_EXPORT_LINE="export PAYARA=\"$PAYARA\""

# Ensure the script is not run as root
if [[ $EUID -eq 0 ]]; then
    printf "Please do not run this script as root.\n" >&2
    printf "This script runs several command with sudo from within functions.\n" >&2
    exit 1
fi

check_current_version() {
    local version response
    response=$(sudo -u dataverse /usr/local/payara/bin/asadmin list-applications)

    # Check if "No applications are deployed to this target server" is part of the response
    if [[ "$response" == *"No applications are deployed to this target server"* ]]; then
        printf " - No applications are deployed to this target server. Assuming upgrade is needed.\n"
        return 0
    fi

    # If no such message, check the Dataverse version via the API
    version=$(curl -s "http://localhost:8080/api/info/version" | grep -oP '\d+\.\d+')

    # Check if the version matches the expected current version
    if [[ $version == "$CURRENT_VERSION" ]]; then
        return 0
    else
        printf " - Current Dataverse version is not %s. Upgrade cannot proceed.\n" "$CURRENT_VERSION" >&2
        return 1
    fi
}

# Function to undeploy the current Dataverse version
undeploy_dataverse() {
    if sudo -u dataverse $PAYARA/bin/asadmin list-applications | grep -q "dataverse-$CURRENT_VERSION"; then
        printf " - Undeploying current Dataverse version...\n"
        sudo -u dataverse $PAYARA/bin/asadmin undeploy dataverse-$CURRENT_VERSION || return 1
    else
        printf " - Dataverse is not currently deployed. Skipping undeploy step.\n"
    fi
}

# Function to stop Payara service
stop_payara() {
    if pgrep -f payara > /dev/null; then
        printf " - Stopping Payara service...\n"
        sudo systemctl stop payara || return 1
    else
        printf " - Payara is already stopped.\n"
    fi
}

stop_solr() {
    if pgrep -f solr > /dev/null; then
        printf " - Stopping solr service...\n"
        sudo systemctl stop solr || return 1
    else
        printf " - solr is already stopped.\n"
    fi
}

start_solr() {
    if ! pgrep -f solr > /dev/null; then
        printf " - Starting solr service...\n"
        sudo systemctl start solr || return 1
    else
        printf " - solr is already running.\n"
    fi
}

# Function to start Payara service
start_payara() {
    if ! pgrep -f payara > /dev/null; then
        printf " - Starting Payara service...\n"
        sudo systemctl start payara || return 1
    else
        printf " - Payara is already running.\n"
    fi
}

# Waiting for payara to come back up.
wait_for_site() {
    local url="https://${DOMAIN}/dataverse/root?q="
    local response_code

    printf " - Waiting for site to become available...\n"
    
    while true; do
        # Get HTTP response code
        response_code=$(curl -o /dev/null -s -w "%{http_code}" "$url")

        if [[ "$response_code" -eq 200 ]]; then
            printf " - Site is up (HTTP 200 OK).\n"
            break
        else
            printf "\r - Waiting... (HTTP response: %s)" "$response_code"
        fi

        # Wait 1 seconds before checking again
        sleep 1
    done
}

# Function to clean generated directory
clean_generated_dir() {
    if [[ -d "$DEPLOY_DIR" ]]; then
        printf " - Removing generated directory...\n"
        sudo rm -rf "$DEPLOY_DIR" || return 1
    else
        printf " - Generated directory already clean. Skipping.\n"
    fi
}

download_war_file() {
    if [[ -f "$DATAVERSE_WAR_FILE" ]]; then
        printf " - WAR file already exists at %s. Skipping download.\n" "$DATAVERSE_WAR_FILE"
        ACTUAL_HASH=$(sudo -u dataverse sha256sum "$DATAVERSE_WAR_FILE" | awk '{print $1}')
        if [ "$ACTUAL_HASH" != "$DATAVERSE_WAR_HASH" ]; then
            echo "Hash mismatch!"
            rm -f $DATAVERSE_WAR_FILE
        else
            echo "Hash matches!"
            return 0
        fi
    fi
    printf " - WAR file not found or it's hash didn't match. Downloading...\n"
    sudo rm -f "$DATAVERSE_WAR_FILE"
    WAR_FILE_NAME=$(basename "$DATAVERSE_WAR_FILE")
    
    if ! sudo -u dataverse bash -c "cd /home/dataverse && curl -L -O \"$DATAVERSE_WAR_URL\""; then
        printf " - Error downloading the WAR file.\n" >&2
        printf " - URL: sudo -u dataverse cd /home/dataverse ; curl -L -O $DATAVERSE_WAR_URL\n"
        return 1
    fi
    printf " - Download completed successfully.\n"
    printf " - URL: sudo -u dataverse cd /home/dataverse ; curl -L -O $DATAVERSE_WAR_URL\n"
    printf " - Setting ownership to $DATAVERSE_WAR_FILE \n"
    sudo chown dataverse:dataverse $DATAVERSE_WAR_FILE
    sudo -u dataverse sha256sum "$DATAVERSE_WAR_FILE" | awk '{print $1}'
    ACTUAL_HASH=$(sudo -u dataverse sha256sum "$DATAVERSE_WAR_FILE" | awk '{print $1}')
    if [ "$ACTUAL_HASH" != "$DATAVERSE_WAR_HASH" ]; then
        echo "$ACTUAL_HASH != $DATAVERSE_WAR_HASH"
        echo "Hash mismatch!"
        return 1
    else
        echo "Hash matches!"
    fi
}

download_geospatial_file() {

    if [[ -f "$GEOSPATIAL_FILE" ]]; then
        printf " - Geospatial file already exists at %s. Skipping download.\n" "$GEOSPATIAL_FILE"
    else
        printf " - Geospatial file not found. Downloading...\n"

        # Ensure the directory exists (it should, as it's /tmp)
        DIRECTORY=$(dirname "$GEOSPATIAL_FILE")
        if [[ ! -d "$DIRECTORY" ]]; then
            printf " - Directory %s does not exist. Creating it...\n" "$DIRECTORY"
            mkdir -p "$DIRECTORY"
        fi

        # Download the file directly to the intended location
        printf " - Attempting to download to %s using URL %s\n" "$GEOSPATIAL_FILE" "$GEOSPATIAL_URL"
        
        # Download the file (removed sudo -u dataverse)
        if curl -L -o "$GEOSPATIAL_FILE" "$GEOSPATIAL_URL"; then
            printf " - Geospatial file downloaded successfully to %s\n" "$GEOSPATIAL_FILE"
        else
            printf " - Error downloading the Geospatial file. Exiting script.\n"
            return 1
        fi

        # Adding a small delay to ensure the file system syncs
        sleep 2

        # Verify if the file exists after a short delay
        if [[ -f "$GEOSPATIAL_FILE" ]]; then
            printf " - Geospatial file verified at %s.\n" "$GEOSPATIAL_FILE"
        else
            printf " - Geospatial file is missing at %s.\n" "$GEOSPATIAL_FILE"
            return 1
        fi
    fi
}

update_geospatial_metadata_block() {
    if [[ -f "$GEOSPATIAL_FILE" ]]; then
        printf " - Geospatial file found. uploading...\n"
        if ! sudo -u dataverse curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file $GEOSPATIAL_FILE ; then
            printf " - Error updating with the Geospatial file.\n" >&2
            return 1
        fi
        printf " - Update completed successfully.\n"
    else
        printf " - Geospatial file is missing at %s.\n" "$GEOSPATIAL_FILE"
        return 1
    fi
    if [[ -f "$GEOSPATIAL_FILE" ]]; then
        rm -f $GEOSPATIAL_FILE
    fi
}

download_citation_file() {
        if [[ -f "$CITATION_FILE" ]]; then
        printf " - Citation file already exists at %s. Skipping download.\n" "$CITATION_FILE"
    else
        printf " - Citation file not found. Downloading...\n"

        # Ensure the directory exists (it should, as it's /tmp)
        DIRECTORY=$(dirname "$CITATION_FILE")
        if [[ ! -d "$DIRECTORY" ]]; then
            printf " - Directory %s does not exist. Creating it...\n" "$DIRECTORY"
            mkdir -p "$DIRECTORY"
        fi

        # Download the file directly to the intended location
        printf " - Attempting to download to %s using URL %s\n" "$CITATION_FILE" "$CITATION_URL"
        
        # Download the file (removed sudo -u dataverse)
        if curl -L -o "$CITATION_FILE" "$CITATION_URL"; then
            printf " - Citation file downloaded successfully to %s\n" "$CITATION_FILE"
        else
            printf " - Error downloading the Citation file. Exiting script.\n"
            return 1
        fi

        # Adding a small delay to ensure the file system syncs
        sleep 2

        # Verify if the file exists after a short delay
        if [[ -f "$CITATION_FILE" ]]; then
            printf " - Citation file verified at %s.\n" "$CITATION_FILE"
        else
            printf " - Citation file is missing at %s.\n" "$CITATION_FILE"
            return 1
        fi
    fi
}

update_citation_metadata_block() {
    if [[ -f "$CITATION_FILE" ]]; then
        printf " - Citation file found. uploading...\n"
        if ! sudo -u dataverse curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file $CITATION_FILE ; then
            printf " - Error updating with the Citation file.\n" >&2
            return 1
        fi
        printf " - Update completed successfully.\n"
    else
        printf " - Citation file is missing at %s.\n" "$CITATION_FILE"
        return 1
    fi
    if [[ -f "$CITATION_FILE" ]]; then
        rm -f $CITATION_FILE
    fi
}

download_solr_schema_updater() {
    if [[ -f "$SOLR_FIELD_UPDATER_FILE" ]]; then
        printf " - Solr Field Updater file already exists at %s. Skipping download.\n" "$SOLR_FIELD_UPDATER_FILE"
    else
        printf " - Solr Field Updater file not found. Downloading...\n"

        # Ensure the directory exists (it should, as it's /tmp)
        DIRECTORY=$(dirname "$SOLR_FIELD_UPDATER_FILE")
        if [[ ! -d "$DIRECTORY" ]]; then
            printf " - Directory %s does not exist. Creating it...\n" "$DIRECTORY"
            mkdir -p "$DIRECTORY"
        fi

        # Download the file directly to the intended location
        printf " - Attempting to download to %s using URL %s\n" "$SOLR_FIELD_UPDATER_FILE" "$SOLR_FIELD_UPDATER_URL"
        
        # Download the file (removed sudo -u dataverse)
        if curl -L -o "$SOLR_FIELD_UPDATER_FILE" "$SOLR_FIELD_UPDATER_URL"; then
            printf " - Solr Field Updater file downloaded successfully to %s\n" "$SOLR_FIELD_UPDATER_FILE"
        else
            printf " - Error downloading the Solr Field Updater file. Exiting script.\n"
            return 1
        fi

        # Adding a small delay to ensure the file system syncs
        sleep 2

        # Verify if the file exists after a short delay
        if [[ -f "$SOLR_FIELD_UPDATER_FILE" ]]; then
            printf " - Solr Field Updater file verified at %s.\n" "$SOLR_FIELD_UPDATER_FILE"
        else
            printf " - Solr Field Updater file is missing at %s.\n" "$SOLR_FIELD_UPDATER_FILE"
            return 1
        fi
    fi
    printf " - Setting permissions to $SOLR_FIELD_UPDATER_FILE \n"
    sudo chmod +x $SOLR_FIELD_UPDATER_FILE
    sudo chown solr: $SOLR_FIELD_UPDATER_FILE
}

update_solr_schema_updater() {
    if [[ -f "$SOLR_FIELD_UPDATER_FILE" ]]; then
        printf " - Solr file found. uploading...\n"
        if ! sudo -u solr bash -c "curl \"http://localhost:8080/api/admin/index/solr/schema\" | bash $SOLR_FIELD_UPDATER_FILE /usr/local/solr/server/solr/collection1/conf/schema.xml" ; then
            printf " - Error updating with the Solr file.\n" >&2
            return 1
        fi
        printf " - Update completed successfully.\n"
    else
        printf " - Solr file is missing at %s.\n" "$SOLR_FIELD_UPDATER_FILE"
        return 1
    fi
    if [[ -f "$SOLR_FIELD_UPDATER_FILE" ]]; then
        sudo rm -f $SOLR_FIELD_UPDATER_FILE
    fi
}

# Function to deploy the new Dataverse WAR
deploy_new_version() {
    if ! sudo -u dataverse $PAYARA/bin/asadmin list-applications | grep -q "dataverse-$TARGET_VERSION"; then
        printf " - Deploying new Dataverse version...\n"
        sudo -u dataverse $PAYARA/bin/asadmin deploy "$DATAVERSE_WAR_FILE" || return 1
    else
        printf " - Dataverse version %s is already deployed. Skipping deployment.\n" "$TARGET_VERSION"
    fi
}

export_all_metadata() {
    sudo -u dataverse curl http://localhost:8080/api/admin/metadata/reExportAll || return 1
}

main() {
    echo "Pre-req: ensure Payara environment variables are set"
    export PAYARA="$PAYARA"
    echo "Checking $BASHRC_FILE for payara export"
    sleep 2
    if ! sudo -u dataverse grep -qF "$PAYARA_EXPORT_LINE" "$BASHRC_FILE"; then
        printf " - Line not found in .bashrc. Adding it...\n"
        sudo bash -c "echo -e '\n$PAYARA_EXPORT_LINE' >> $BASHRC_FILE"
        printf " - Line added to .bashrc.\n"
    else
        printf " - Line already exists in .bashrc. Skipping addition.\n"
    fi

    echo "Check if Dataverse is running the correct version"
    sleep 2
    if ! check_current_version; then
        printf " - Failed to find $CURRENT_VERSION deployed.\n" >&2
        exit 1
    fi

    echo -e "\nStep 1: Undeploy the existing version"
    sleep 2
    if ! undeploy_dataverse; then
        printf " - Step 1: Error during undeploy.\n" >&2
        exit 1
    fi

    echo -e "\nStep 2: Stop Payara and clean directories"
    sleep 2
    if ! stop_payara || ! clean_generated_dir; then
        printf " - Step 2: Error stopping Payara or cleaning generated directories.\n" >&2
        exit 1
    fi

    echo -e "\nStep 3: Start Payara and deploy the new version"
    sleep 2
    if ! start_payara; then
        printf " - Step 3: Error starting Payara.\n" >&2
        exit 1
    fi

    echo -e "\nStep 4: download war file."
    sleep 2
    if ! download_war_file; then
        printf " - Step 4: Failed to download WAR file. Exiting script.\n" >&2
        exit 1
    fi
    echo -e "\nStep 4: deploying war file."
    if ! deploy_new_version; then
        printf " - Step 4: Error deploying new version.\n" >&2
        exit 1
    fi

    echo -e "\nStep 5: Restart Payara"
    sleep 2
    if ! stop_payara || ! start_payara; then
        printf " - Step 5: Error restarting Payara after deployment.\n" >&2
        exit 1
    fi

    echo -e "\nWait for Payara to come up."
    wait_for_site

    echo -e "\nStep 6: Update Geospatial Metadata Block (to improve validation of bounding box values)"
    sleep 2
    if ! download_geospatial_file; then
        printf " - Step 6: Failed to download geospatial file. Exiting script.\n" >&2
        exit 1
    fi
    if ! update_geospatial_metadata_block; then
        printf " - Step 6: Failed to update geospatial metadata block. Exiting script.\n" >&2
        exit 1
    fi

    echo -e "\nStep 6a: Update Citation Metadata Block (to make Alternative Title repeatable)"
    sleep 2
    if ! download_citation_file; then
        printf " - Step 6a: Failed to download citation file. Exiting script.\n" >&2
        exit 1
    fi
    if ! update_citation_metadata_block; then
        printf " - Step 6a: Failed to update citation metadata block. Exiting script.\n" >&2
        exit 1
    fi

    echo -e "\nStep 7b: Upate Solr schema.xml to allow multiple Alternative Titles to be used. For installations with custom or experimental metadata blocks."
    sleep 2
    if ! stop_solr; then
        printf " - Step 7b: Error stopping solr.\n" >&2
        exit 1
    fi
    if ! download_solr_schema_updater; then
        printf " - Step 7b: Failed to download Solr's schema. Exiting script.\n" >&2
        exit 1
    fi
    if ! update_solr_schema_updater; then
        printf " - Step 7b: Failed to update citation metadata block. Exiting script.\n" >&2
        exit 1
    fi
    if ! start_solr; then
        printf " - Step 7b: Error starting solr.\n" >&2
        exit 1
    fi

    echo -e "\nStep 8: Run ReExportAll to update dataset metadata exports. Follow the directions in the Admin Guide."
    sleep 2
    if ! export_all_metadata; then
        printf " - Step 8: Error exporting all metadata.\n" >&2
        exit 1
    fi
    sudo yum install -y ImageMagick
    printf "Add <jvm-options>-Dconvert.path=/usr/bin/convert</jvm-options> to the domain.xml file"
    printf "\nUpgrade to Dataverse %s completed successfully.\n\n\n" "$TARGET_VERSION"
}

# Run the main function
main "$@"