download-smartling-files.sh

gistfile1.sh

#!/bin/bash

#Dependencies
#Parallel with Homebrew: https://github.com/Homebrew/homebrew/blob/master/Library/Formula/parallel.rb

#Usage: 
#./download-smartling-files.sh <file_type> <api_key> <project_id>

shopt -s nullglob

#Test Glob
if test -n "$(find . -maxdepth 1 -name 'glob*' -print -quit)"
then
echo found
else
echo not found
fi

if [ $# -lt 3 ] ; then
echo "USAGE: download-smartling-files TYPE (original | published | pseudo | pending) APIKEY PROJECTID [Optional: URI-MASK].   Exiting."
exit
fi
DL_TYPE=$1
SL_APIKEY=$2
SL_PROJECT=$3
SERVER_URL=api.smartling.com
PARALLEL_DOWNLOADS=20

MASK=0
URI_MASK=""
if [ $# -eq 4 ] ; then
    URI_MASK=$4
    MASK=1
    echo "File mask specified!"
    else
        echo "No mask - all files will be downloaded"
fi

# make sure download type is valid
if [ "$DL_TYPE" != "original" ] && [ "$DL_TYPE" != "published" ] && [ "$DL_TYPE" != "pseudo" ] && [ "$DL_TYPE" != "pending" ] ; then
    echo "INVALID TYPE. Acceptable types: original | published | pseudo | pending.  Exiting."
    exit
fi

#oIFS=$IFS
IFS=$'\n'

# get just the fileURI key values then clean it up with sed to just get the actual URI
# in order to keep SPACES in filename which are common we change IFS to newline
uris=($(curl -s -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&uriMask=$URI_MASK" "https://$SERVER_URL/v1/file/list" |  grep -Eo "fileUri\":\"[^\"]*" | sed -e 's/fileUri":"//g'))

# exit if nothing matched at all, otherwise show the count of files that matched and the names
file_count=${#uris[@]}
if [ $file_count = "0" ]; then echo "nothing to download, exiting!"; exit; fi
echo "files to download: "$file_count$'\n'
echo ${uris[@]}

if [ "$DL_TYPE" == "original" ] ; then
    echo "downloading originals"

# create the originals folder if it doesn't exist
    if [ ! -d "originals" ]; then
        echo "making originals folder"$'\n'
        mkdir originals
    fi

    COUNT=0
    # go through the list of files to download (in each language)
    for CURRENT_FILE in ${uris[@]}
    do
        echo $CURRENT_FILE
        ((COUNT++))
        echo $COUNT

        # since the URI includes full URI including default smartling prefix '/files/' or any other prefix specified - strip that and just use the last part after the last /
        # e.g. if the URI is /files/filename.ext then BF_NAME will be filename.ext - this what we use for the local file name (in the locale folder)
        BF_NAME=${CURRENT_FILE##*/}
        # This is curl call that downloads the originals. Since no LOCALE is set that is the behavior of /file/get

        # becuase we are flattening the URIs to basename it's possible filenames could be duplicated - so to avoid this check if the file already exist and if it does use count to append name to create unique
        # TODO USE COUNT
        if [ -e originals/$BF_NAME ]; then
            BF_NAME=$COUNT.$BF_NAME
            echo "Updated BF_NAME"
        fi

        curl -s -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT&fileUri=$CURRENT_FILE" "https://$SERVER_URL/v1/file/get" > originals/$BF_NAME
        echo "downloaded URI: "$CURRENT_FILE" to: originals/"$BF_NAME$'\n'

    done
exit
fi
#end of originals


#if not getting originals - then getting translations so get locales and then use that to download all
# get the list of locales for this project.  Similar to above grep out the locale key value then sed for just the value

LOCALES=($(curl -s -d "apiKey=$SL_APIKEY&projectId=$SL_PROJECT" "https://$SERVER_URL/v1/project/locale/list" |  grep -Eo "locale\":\"[^\"]*" | sed -e 's/locale":"//g'))

echo "locales: "${#LOCALES[@]}$'\n'
# count of the locales
echo ${LOCALES[@]}$'\n'

#script puts the translated' versions in 'translated folder' with sub-folders for locales
if [ ! -d "translated" ]; then
    echo "making translated folder"$'\n'
    mkdir translated
fi

# make the subfolders in the translated folder if they don't exist
for CURRENT_LOCALE in ${LOCALES[@]}
    do
    if [ ! -d translated/$CURRENT_LOCALE ]; then
        echo "making "$CURRENT_LOCALE" folder" $'\n'
        mkdir translated/$CURRENT_LOCALE
    fi
done

# go through the list of files to download (in each language)

COUNT=0
for CURRENT_FILE in ${uris[@]}
do
    echo 'echo "'$CURRENT_FILE'"'
    ((COUNT++))
    echo 'echo "'$COUNT'"'
    # loop through all the locales for each file

#    IFS=$oIFS
    for CURRENT_LOCALE in ${LOCALES[@]}
    do

        # since the URI includes full URI including default smartling prefix '/files/' or any other prefix specified - strip that and just use the last part after the last /
        # e.g. if the URI is /files/filename.ext then BF_NAME will be filename.ext - this what we use for the local file name (in the locale folder)
        BF_NAME=${CURRENT_FILE##*/}

        #the "right" way to do this would be to check the response header but since returns filename with URI which I want to avoid in the first place just do simple file extension check
        EXT=${BF_NAME##*.}
        if [ $EXT = "pot" ]; then
            filename=${BF_NAME%.*}
            BF_NAME=$filename".po"
        #echo "renamed pot to po"
        fi

        # becuase we are flattening the URIs to basename it's possible filenames could be duplicated - so to avoid this check if the file already exist and if it does use count to append name to create unique
        # TODO USE COUNT
        if [[ -e translated/$CURRENT_LOCALE/$BF_NAME ]]; then
            BF_NAME=$COUNT.$BF_NAME
            echo 'echo "Updated '$BF_NAME'"'
        fi

        # This is the actual curl command that downloads the given file for  given locale, and state specified on the command line
        echo 'curl --compressed -H "Accept-Encoding: gzip" -s -d "apiKey='$SL_APIKEY'&projectId='$SL_PROJECT'&fileUri='$CURRENT_FILE'&locale='$CURRENT_LOCALE'&retrievalType='$DL_TYPE'" "https://'$SERVER_URL'/v1/file/get" > translated/'$CURRENT_LOCALE'/'$BF_NAME'&& echo "downloaded URI: '$CURRENT_FILE' to: translated/'$CURRENT_LOCALE'/'$BF_NAME'"\n'

    done

done | parallel --group -j $PARALLEL_DOWNLOADS

exit