Created
August 1, 2024 20:04
-
-
Save h8rt3rmin8r/f9d1fc10c1a3018d44066b3f6d7bb9e8 to your computer and use it in GitHub Desktop.
Download the latest data for the Tamriel Trade Centre addon (for the Elder Scrolls Online game) and generate a historical archive of price data with each subsequent download.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #! /usr/bin/env bash | |
| #>------------------------------------------------------------------------------ | |
| #> | |
| #> [ ttcUpdate.sh ] | |
| #> | |
| #> Download the latest data for the Tamriel Trade Centre addon and generate | |
| #> a historical archive of price data with each subsequent download. | |
| #> | |
| #> No inputs are required when invoking this script. | |
| #> | |
| #> Created on 20210603 by h8rt3rmin8r | |
| #> Updated on 20210619 by h8rt3rmin8r (v. 1624085579) | |
| #> Updated on 20210809 by h8rt3rmin8r (v. 1628515982) | |
| #> - Integrated ttcParse operations for archiving price history | |
| #> - Added automatic logging | |
| #> | |
| #> USAGE: | |
| #> | |
| #> ttcUpdate.sh <INPUT> | |
| #> ttcUpdate.sh <OPTION> | |
| #> ttcUpdate.sh <INPUT> <OPTION> | |
| #> | |
| #> where "INPUT" is an optional directory reference for output data to be | |
| #> stored; and where "OPTION" is an optional input parameter; and where | |
| #> "OPTION" is one of the following: | |
| #> | |
| #> | | |
| #> -f, --force | Force the script to update TTC data even if the game | |
| #> | is currently running (NOT RECOMMENDED) | |
| #> | | |
| #> -h, --help | Print this help text to the terminal | |
| #> | | |
| #> -s, --silent | Suppress verbosity messages | |
| #> | | |
| #> -v, --verbose | Print verbosity messages (default) | |
| #> | | |
| #> -V, --version | Print the script version information | |
| #> | | |
| #> | |
| #> REFERENCE: | |
| #> | |
| #> # Official website for TamrielTradeCentre | |
| #> https://tamrieltradecentre.com/ | |
| #> | |
| #> # Addon Page for TamrielTradeCentre | |
| #> https://www.esoui.com/downloads/info1245-TamrielTradeCentre.html | |
| #> | |
| #>------------------------------------------------------------------------------ | |
| #> VERSION: 1628857161 | |
| #> | |
| #_______________________________________________________________________________ | |
| # Declare functions | |
| function ttcParse_archiver() { | |
| local proc_success=0 | |
| local final_archive="${out_dir_parse_archive}/${d_t_string}.json" | |
| for i in "${parse_keys[@]}"; do | |
| local cycle_key="${i}" | |
| local cycle_key_mod="${cycle_key,,}" | |
| local cycle_master="${sh_dir}/prices-${cycle_key}.json" | |
| local cycle_archive="${out_dir_parse_archive}/${d_t_string}-${cycle_key_mod}" | |
| if [[ -f "${cycle_master}" ]]; then | |
| cat "${cycle_master}" > "${cycle_archive}" | |
| local proc_success=1 | |
| fi | |
| done | |
| if [[ "${proc_success}" -eq 1 ]]; then | |
| ## if archival files were created in the previous step, combine them into one file | |
| local cycle_counter=0 | |
| printf '%s' "${_cba}" > "${final_archive}" | |
| for i in "${parse_keys[@]}"; do | |
| local cycle_key="${i}" | |
| local cycle_key_mod="${cycle_key,,}" | |
| local cycle_part="${out_dir_parse_archive}/${d_t_string}-${cycle_key_mod}" | |
| if [[ -f "${cycle_part}" ]]; then | |
| ## if a file exists for the current parse key, add its contents to the final archive file | |
| if [[ "${cycle_counter}" -gt 0 ]]; then | |
| printf '%s' ",${_q2}${cycle_key_mod}${_q2}:" >> "${final_archive}" | |
| cat "${cycle_part}" \ | |
| | tr -d '\n' >> "${final_archive}" | |
| else | |
| printf '%s' "${_q2}${cycle_key_mod}${_q2}:" >> "${final_archive}" | |
| cat "${cycle_part}" \ | |
| | tr -d '\n' >> "${final_archive}" | |
| fi | |
| ## delete the partial archive file used in this cycle | |
| rm "${cycle_part}" | |
| let cycle_counter=cycle_counter+1 | |
| fi | |
| done | |
| printf '%s' "${_cbb}" >> "${final_archive}" | |
| gzip -9 "${final_archive}" | |
| fi | |
| return $? | |
| } | |
| function ttcParse_logger() { | |
| local out_string="${_q2}${d_t_string}${_q2},${_q2}${d_t_local}${_q2},${_q2}${d_t_weekday}${_q2}" | |
| echo "${out_string}" >> "${out_file_runhistory}" | |
| return $? | |
| } | |
| function ttcParse_make() { | |
| function ttcParse_make_dirs() { | |
| mkdir -p "${out_dir_parse}" | |
| mkdir -p "${out_dir_parse_archive}" | |
| return $? | |
| } | |
| function ttcParse_make_files() { | |
| if [[ ! -f "${out_file_notfound}" ]]; then | |
| echo "${out_columns_notfound}" > "${out_file_notfound}" | |
| fi | |
| if [[ ! -f "${out_file_runhistory}" ]]; then | |
| echo "${out_columns_runhistory}" > "${out_file_runhistory}" | |
| fi | |
| return $? | |
| } | |
| ttcParse_make_dirs | |
| ttcParse_make_files | |
| return $? | |
| } | |
| function ttcParse_run() { | |
| function ttcParse_run_python() { | |
| if [[ "x${1}" == "x" ]]; then | |
| return 1 | |
| fi | |
| echo -e "import re\nimport json\ntranslation = {}\nprices = {}\nfil = open(\"PriceTable.lua\")\ntext = fil.read()\nfil.close()\noutputsL = re.findall(r'\[([0-9]*?)\]={\[[0-5]]={\[[0-9]*?\]={\[-1\]={.*?\"$1\"]=(.*?),',text)\nfil = open(\"ItemLookUpTable_EN.lua\")\ntext = fil.read()\nfil.close()\ntranslationL = re.findall(r'\[\"(.*?)\"\]={\[[0-9]*?]=(.[0-9]*?),}', text)\nfor x in translationL: translation[x[1]] = x[0]\nfor x in outputsL: \n\ttry: prices[translation[x[0]]] = x[1]\n\texcept KeyError: print(f\"Item {x} not found\")\nfil = open(\"prices-$1.json\",'w')\njson.dump(prices, fil)\nfil.close()" \ | |
| | python3 | |
| } | |
| cd "${out_dir}" | |
| ttcParse_make | |
| for i in "${parse_keys[@]}"; do | |
| ttcParse_run_python "${i}" \ | |
| | sed "s/^/\"$d_t\",\"$i\",\"/" \ | |
| | sed 's/$/\"/' >> "${out_file_notfound}" | |
| done | |
| ttcParse_archiver | |
| ttcParse_logger | |
| cd "${here_now}" | |
| return $? | |
| } | |
| function ttcParse_unixtime2date() { | |
| # USAGE: | |
| # | |
| # ttcParse_unixtime2date <OPTION> <INPUT> | |
| # | |
| # where "INPUT" is a valid Unix timestamp and where "OPTION" is one of the | |
| # following: | |
| # | |
| # | | |
| # -D, --day | Output only the name of the related weekday | |
| # | | |
| # -H, --human | Output is formatted in human-friendly format (default) | |
| # | (does NOT preserve nano-seconds) | |
| # | | |
| # -i, --iso | Output is formatted as an ISO date: YYYY-MM-DD HH:MM:SS | |
| # | (preserves nano-seconds) | |
| # | | |
| if [[ "x${1}" == "x" ]]; then | |
| return 1 | |
| fi | |
| local e_c="0" | |
| if [[ "${1}" =~ ^[-]+[dDhHiInNsStTzZ] ]]; then | |
| local i_n="${1//[^a-zA-Z]}" | |
| case "${i_n}" in | |
| D|day|weekday) | |
| shift 1 | |
| if (($#)); then | |
| local in_x="$@" | |
| local DT=${in_x:0:10} | |
| local output=$(printf "%(%A)T\n" "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| echo "${output}" | |
| else | |
| while read -r line; do | |
| local in_x="${line}" | |
| local DT=${in_x:0:10} | |
| local output=$(printf "%(%A)T\n" "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| echo "${output}" | |
| done | |
| fi | |
| if [[ "${e_c}" =~ 1 ]]; then | |
| return 1 | |
| else | |
| return 0 | |
| fi | |
| ;; | |
| H|human) | |
| shift 1 | |
| if (($#)); then | |
| local in_x="$@" | |
| local DT=${in_x:0:10} | |
| local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| echo "${output}" | |
| else | |
| while read -r line; do | |
| local DT=${line:0:10} | |
| local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| echo "${output}" | |
| done | |
| fi | |
| if [[ "${e_c}" =~ 1 ]]; then | |
| return 1 | |
| else | |
| return 0 | |
| fi | |
| ;; | |
| i|iso) | |
| shift 1 | |
| if (($#)); then | |
| local in_x="$@" | |
| local DT=${in_x:0:10} | |
| local DT_X=${in_x#$DT} | |
| local out_pre=$(printf "%(%Y-%m-%d %H:%M:%S)T\n" "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| if [[ "x${DT_X}" == "x" ]]; then | |
| local output="${out_pre}" | |
| else | |
| local output="${out_pre}.${DT_X}" | |
| fi | |
| echo "${output}" | |
| else | |
| while read -r line; do | |
| local in_x="${line}" | |
| local DT=${in_x:0:10} | |
| local DT_X=${in_x#$DT} | |
| local out_pre=$(printf "%(%Y-%m-%d %H:%M:%S)T\n" "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| if [[ "x${DT_X}" == "x" ]]; then | |
| local output="${out_pre}" | |
| else | |
| local output="${out_pre}.${DT_X}" | |
| fi | |
| echo "${output}" | |
| done | |
| fi | |
| if [[ "${e_c}" =~ 1 ]]; then | |
| return 1 | |
| else | |
| return 0 | |
| fi | |
| ;; | |
| esac | |
| fi | |
| if (($#)); then | |
| local in_x="$@" | |
| local DT=${in_x:0:10} | |
| local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| echo "${output}" | |
| else | |
| while read -r line; do | |
| local DT=${line:0:10} | |
| local output=$(printf '%(%c)T\n' "${DT}" 2>/dev/null) | |
| local e_c="${e_c}$?" | |
| echo "${output}" | |
| done | |
| fi | |
| if [[ "${e_c}" =~ 1 ]]; then | |
| return 1 | |
| else | |
| return 0 | |
| fi | |
| } | |
| function ttcUpdate_help() { | |
| # Help text printing function | |
| cat "${sh_path}" 2>/dev/null \ | |
| | grep -E '^#[>]' \ | |
| | sed 's/^..//' | |
| return $? | |
| } | |
| function ttcUpdate_report() { | |
| # Runtime reporting function | |
| function ttcUpdate_report_proctime() { | |
| # Calculate the total script execution time | |
| local out_prefix=$(echo "${time_done} - ${time_start}" | bc) | |
| local out_string="${out_prefix} seconds" | |
| ## print the final result and exit the function | |
| printf '%s\n' "${out_string}" | |
| return $? | |
| } | |
| if [[ "${vbs_ops}" -ne 0 ]]; then | |
| ## skip the final runtime reporting if running in silent mode | |
| return 0 | |
| fi | |
| export time_done="$(date '+%s.%N')" | |
| local time_proc="$(ttcUpdate_report_proctime)" | |
| ttcUpdate_vbs "Final runtime report:" | |
| echo " Operations Begin (UNIX): ${time_start}" &>${vbs_out} | |
| echo " Operations End (UNIX): ${time_done}" &>${vbs_out} | |
| echo " Execution Time (seconds): ${time_proc}" &>${vbs_out} | |
| unset time_done | |
| unset time_start | |
| unset time_proc | |
| return $? | |
| } | |
| function ttcUpdate_run() { | |
| # Main script operations function | |
| ## Declare sub-functions | |
| function ttcUpdate_run_extract() { | |
| # Extract the newly downloaded TTC data archive | |
| local e_c=0 | |
| cd "${out_dir}" 2>/dev/null | |
| local e_c="$?" | |
| if [[ "${e_c}" -ne 0 ]]; then | |
| ## kill the function if we can't get into the target directory for some unknown reason | |
| return $e_c | |
| fi | |
| if [[ "${vbs_ops}" -eq 0 ]]; then | |
| ## verbosity is currently ON | |
| unzip -o "${out_file}" | |
| local e_c="$?" | |
| else | |
| ## verbosity is currently OFF | |
| unzip -o -q "${out_file}" | |
| local e_c="$?" | |
| fi | |
| cd "${here_now}" | |
| return $e_c | |
| } | |
| function ttcUpdate_run_get() { | |
| # Fetch the latest TTC data from the remote server | |
| local e_c="" | |
| if [[ "${vbs_ops}" -eq 0 ]]; then | |
| ## display a progress bar in the terminal while the data is being downloaded | |
| curl --xattr -o "${out_file}" "${api_target}" | |
| local e_c=$? | |
| else | |
| ## suppress verbosity if the script was invoked with the "silent" parameter | |
| curl -s --xattr -o "${out_file}" "${api_target}" | |
| local e_c=$? | |
| fi | |
| return $e_c | |
| } | |
| ## Execute operations | |
| local e_c="" | |
| ttcUpdate_vbs "Updating reference data for the Tamriel Trade Centre addon" | |
| ttcUpdate_vbs "Destination directory: ${out_dir}" | |
| if [[ "${force_ops}" -ne 0 ]]; then | |
| ## print a warning if running with "force" | |
| ttcUpdate_vbs "WARNING: Running in forced execution mode! (This could result in corrupted data)" | |
| fi | |
| ttcUpdate_vbs "Beginning download ..." | |
| ttcUpdate_run_get | |
| local e_c="$?" | |
| if [[ "${e_c}" -ne 0 ]]; then | |
| ## download process failed | |
| ttcUpdate_vbs "ERROR: An unknown error occurred while communicating with the remote host: ${api_target} (exit code: ${e_c})" | |
| ttcUpdate_vbs "ERROR: Check your network connection and try again." | |
| ttcUpdate_vbs "WARNING: Terminating operations (no changes were made to the system files)" | |
| rm "${out_file}" 2>/dev/null | |
| return $e_c | |
| fi | |
| ttcUpdate_vbs "Download complete (exit code: ${e_c})" | |
| ttcUpdate_vbs "New archive location: ${out_file}" | |
| ttcUpdate_vbs "Extracting new data archive ..." | |
| ttcUpdate_run_extract | |
| local e_c="$?" | |
| if [[ "${e_c}" -ne 0 ]]; then | |
| ## archive extraction failed | |
| ttcUpdate_vbs "ERROR: An unknown error occurred while attempting to extract the archive: ${out_file} (exit code: ${e_c})" | |
| ttcUpdate_vbs "ERROR: The downloaded data might be corrupted. Check your network connection and try again." | |
| ttcUpdate_vbs "WARNING: Terminating operations (no changes were made to the system files)" | |
| rm "${out_file}" 2>/dev/null | |
| return $e_c | |
| fi | |
| ttcUpdate_vbs "Archive extraction complete (exit code: ${e_c})" | |
| ttcUpdate_vbs "All update operations are COMPLETE" | |
| return $e_c | |
| } | |
| function ttcUpdate_vbs() { | |
| # Verbosity handling function | |
| local vbs_i_n="$@" | |
| local vbs_d_t="$(date '+%s.%N')" | |
| local vbs_p_x="${sh_ppid}" | |
| local vbs_n_m="${sh_name}" | |
| echo "${vbs_d_t}|${vbs_p_x}|${vbs_n_m}|${vbs_i_n}" &>${vbs_out} | |
| return $? | |
| } | |
| function ttcUpdate_version() { | |
| # Script version printing function | |
| local e_c="" | |
| local out_string="" | |
| local ver_string=$(cat "${sh_path}" 2>/dev/null | grep -E '^#[>]' | grep -E ' *VERSION. *1([0-9]){9} ?$' | sed 's/[^0-9]*//g' | tail -n 1) | |
| ## construct the output string | |
| if [[ "x${ver_string}" == "x" ]]; then | |
| local out_string="${sh_file} (version not detected)" | |
| local e_c=1 | |
| else | |
| local out_string="${sh_file} v.${ver_string}" | |
| local e_c=0 | |
| fi | |
| ## print the output string and kill the function | |
| printf '%s\n' "${out_string}" &>/dev/stdout | |
| return $e_c | |
| } | |
| #_______________________________________________________________________________ | |
| # Declare variables and arrays | |
| _q1="'" | |
| _q2='"' | |
| _cba='{' | |
| _cbb='}' | |
| time_start="$(date '+%s.%N')" | |
| time_done="" | |
| d_t="${time_start}" | |
| d_t_string="${d_t//.}" | |
| d_t_local=$(ttcParse_unixtime2date --iso "${d_t_string}") | |
| d_t_weekday=$(ttcParse_unixtime2date --day "${d_t_string}") | |
| here_now="${PWD}" | |
| e_c_main=0 | |
| vbs_out="/dev/stderr" | |
| vbs_ops=0 | |
| out_ops=0 | |
| force_ops=0 | |
| sh_path=$(readlink -f "$0") | |
| sh_dir="${sh_path%\/*}" | |
| sh_file="${sh_path//*\/}" | |
| sh_name="${sh_file%.*}" | |
| sh_ppid="${PPID}" | |
| eso_active=$(pgrep -l eso | grep --color=never -E 'eso((6|3)(4|2))[.]exe' &>/dev/null; echo $?) | |
| api_target="https://us.tamrieltradecentre.com/download/PriceTable" | |
| out_dir="${sh_dir}" | |
| out_dir_parse="${out_dir}/ttcParse" | |
| out_dir_parse_archive="${out_dir_parse}/prices_archive" | |
| out_file="${sh_dir}/PriceTable.zip" | |
| out_file_notfound="${out_dir_parse}/notfound.csv" | |
| out_file_runhistory="${out_dir_parse}/runhistory.csv" | |
| out_columns_notfound="${_q2}runTime${_q2},${_q2}missingKey${_q2},${_q2}message${_q2}" | |
| out_columns_runhistory="${_q2}unixTime${_q2},${_q2}localTime${_q2},${_q2}weekDay${_q2}" | |
| declare -a parse_keys=( Avg Max Min EntryCount AmountCount ) | |
| #_______________________________________________________________________________ | |
| # Execute operations | |
| ## catch inputs and parameters | |
| while [[ "$#" -gt 0 ]]; do | |
| sh_in="${1}" | |
| sh_in_mod="${sh_in//-}" | |
| case "${sh_in_mod}" in | |
| f|F|force) | |
| ## set the script to force-run (NOT RECOMMENDED) | |
| force_ops=1 | |
| shift 1 | |
| ;; | |
| h|H|help) | |
| ## print the help text and kill the script | |
| ttcUpdate_help | |
| exit $? | |
| ;; | |
| s|silent|q|quiet) | |
| ## suppress verbosity messages | |
| vbs_out="/dev/null" | |
| vbs_ops=1 | |
| shift 1 | |
| ;; | |
| v|verbose) | |
| ## enable verbosity messages (default) | |
| vbs_out="/dev/stderr" | |
| vbs_ops=0 | |
| shift 1 | |
| ;; | |
| V|version) | |
| ## print the script version number and kill the script | |
| ttcUpdate_version | |
| exit $? | |
| ;; | |
| *) | |
| ## check if the input is a valid directory reference | |
| sh_in_dir=$(readlink -f "${sh_in}") | |
| if [[ -d "${sh_in_dir}" ]]; then | |
| ## reconfigure the output directory locations to the location specified and set related ops code | |
| out_ops=1 | |
| out_dir="${sh_in_dir}" | |
| out_dir_parse="${out_dir}/ttcParse" | |
| out_dir_parse_archive="${out_dir_parse}/prices_archive" | |
| out_file="${out_dir}/PriceTable.zip" | |
| out_file_notfound="${out_dir_parse}/notfound.csv" | |
| out_file_runhistory="${out_dir_parse}/runhistory.csv" | |
| else | |
| ## input was not a valid directory reference | |
| ## warn the user about an unknown input and kill the script | |
| ttcUpdate_vbs "WARNING: Unrecognized input parameter detected: ${1} (terminating operations)" | |
| ttcUpdate_vbs "Use '--help' for more information" | |
| exit 1 | |
| fi | |
| shift 1 | |
| ;; | |
| esac | |
| done | |
| ## check if ESO is currently running and process "force" parameters if necessary | |
| if [[ "${eso_active}" -eq 0 ]]; then | |
| ## ESO IS ACTIVE | |
| ## determine if we should force the script to run anyway ... | |
| if [[ "${force_ops}" -eq 0 ]]; then | |
| ## user didn't request to force-run the script | |
| ## exit the script to avoid data corruption | |
| ttcUpdate_vbs "ERROR: Cannot update addon data while ESO is running" | |
| ttcUpdate_vbs "Exit the game and try again (or force execution with '--force')" | |
| exit 1 | |
| fi | |
| ## the user wants to force the script to run, so let's keep going ... | |
| fi | |
| ## run the core script operations | |
| ttcUpdate_run | |
| e_c_main="$?" | |
| ttcParse_run | |
| ttcUpdate_report | |
| exit $e_c_main |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment