한경컨센서스에서 보고서 다운로드 받기

dl_hankyung.sh

#!/bin/bash -e

TODAY=$(date '+%Y-%m-%d')
DOWNLOAD_DIR="${HOME}/Downloads/${TODAY}"
mkdir -p "${DOWNLOAD_DIR}"

type qpdf &>/dev/null || brew install qpdf
type recode &>/dev/null || brew install recode

curl  -H 'Accept-Charset: euc-kr' --silent "http://consensus.hankyung.com/apps.analysis/analysis.list?&pagenum=800" | iconv -f euc-kr -t utf-8 | grep report_idx | while IFS= read -r LINE; do
    TITLE=$(echo "${LINE}" | sed -E 's/.*title=\"([^\"]*).*/\1/p' | uniq)
    if [[ $TITLE =~ .*\&#.* ]]; then
        TITLE="$(echo ${TITLE} | recode html..utf8)"
    fi
    HREF=$(echo "${LINE}" | sed -E 's/.*href=\"([^\"]*).*/\1/p' | uniq)
    echo "TITLE=${TITLE}"
    echo "HREF=$HREF"

    wget -b -q -c --content-disposition --compression=none -o /tmp/wget-log -O "${DOWNLOAD_DIR}/${TITLE}" http://consensus.hankyung.com${HREF}
done