drupal exports webforms daily

backfill.sh

#!/bin/bash
# Export historical webform data -- some variables assume an acquia environment -- adjust as needed

cd /var/www/html/"$AH_SITE_GROUP"."$AH_SITE_ENVIRONMENT"/docroot/

DATE_COMMAND="date"  # on mac, change to gdate from homebrew's 'coreutil' package

LOCATION="/var/www/html/$AH_SITE_GROUP.$AH_SITE_ENVIRONMENT/docroot/sites/default/files/webform_exports"

today=$($DATE_COMMAND +"%Y-%m-%d %H:%M:%S")
today_filename=$($DATE_COMMAND +"%Y-%m-%d")
today_ym=$($DATE_COMMAND +"%Y-%m")
end_formatted_date=$($DATE_COMMAND -d "$today" +"%A, %B %d, %Y %I:%M:%S %p")

seven_days_ago=$($DATE_COMMAND -d "7 days ago" +"%Y-%m-%d")
start_formatted_date=$($DATE_COMMAND -d "$seven_days_ago" +"%A, %B %d, %Y %I:%M:%S %p")

#echo mkdir -p "$LOCATION"/"$today_ym"



forms=('webform1' 'webform2' 'webform3')
for form in "${forms[@]}"; do
  start_date="2023-01-01"
  end_date="2023-08-11"
  current_date="$start_date"
  while [[ "$current_date" < "$end_date" ]]; do
#      "$current_date"
      script_start=$current_date
      current_date=$(date -d "$current_date + 7 days" +"%Y-%m-%d")
      script_end=$current_date
      current_ym=$(date -d "$current_date" "+%Y-%m")
      mkdir -p "$LOCATION"/"$current_ym"
      if [[ ! -f "${LOCATION}/${current_ym}/${form}-${script_end}.tar.gz" ]]; then
          #echo "${LOCATION}/${current_ym}/${form}-${script_end}.tar.gz"
          vendor/bin/drush webform:export --exporter=json --archive-type=tar --range-type=date --range-start="$script_start" --range-end="$script_end" --destination="$LOCATION"/"$current_ym"/"$form"-"$script_end".tar.gz "$form"
      else
         echo "Skipped: " "$LOCATION"/"$current_ym"/"$form"-"$script_end".tar.gz
      fi
  done
done

daily-export.bash

#!/bin/bash
# Export webform data -- some variables assume an acquia environment -- adjust as needed

if [ "$AH_SITE_ENVIRONMENT" == "LANDO" ]; then
    cd /app/docroot/
    LOCATION="/app/docroot/sites/default/files/webform_exports"
else
    cd /var/www/html/"$AH_SITE_GROUP"."$AH_SITE_ENVIRONMENT"/docroot/
    LOCATION="/var/www/html/$AH_SITE_GROUP.$AH_SITE_ENVIRONMENT/docroot/sites/default/files/webform_exports"
fi

DATE_COMMAND="date"  # on mac, change to gdate from homebrew's 'coreutil' package

today=$($DATE_COMMAND +"%Y-%m-%d %H:%M:%S")
today_filename=$($DATE_COMMAND +"%Y-%m-%d")
today_ym=$($DATE_COMMAND +"%Y-%m")
end_formatted_date=$($DATE_COMMAND -d "$today" +"%A, %B %d, %Y %I:%M:%S %p")

seven_days_ago=$($DATE_COMMAND -d "7 days ago" +"%Y-%m-%d")
start_formatted_date=$($DATE_COMMAND -d "$seven_days_ago" +"%A, %B %d, %Y %I:%M:%S %p")

mkdir -p "$LOCATION"/"$today_ym"

forms=('webform1' 'webform2' 'webform3')
for form in "${forms[@]}"; do
  vendor/bin/drush webform:export --exporter=json --archive-type=tar --range-type=date --range-start="$start_formatted_date" --range-end="$end_formatted_date" --destination="$LOCATION"/"$today_ym"/"$form"-"$today_filename".tar.gz "$form"
done

working-snippets.bash

#!/bin/bash

# Extract all submission tar.gz files, 
find . -name '*tar.gz' -exec tar xf {} \;

# Combine all submission json files into a submissions.jsonl file
cat submission-*.json >> submissions.jsonl

# if the above returns an error like, 
#   exec: Failed to execute process '/bin/ls': the total size of the argument list and exported variables (2.7MB) exceeds the OS limit of 1MB.
for file in ./submission-*.json; do cat $file >> submissions.jsonl; echo "" >> submissions.jsonl; done