crawl baidu music by top500/new100/artist/songid

crawl_baidu_music.py

#!/usr/bin/env python
# coding: utf-8

import gflags
import logging
import os
import sys

from pygaga.helpers.logger import log_init
from pygaga.helpers.utils import make_dirs_for_file
from pygaga.helpers.urlutils import download, DEFAULT_UA, parse_html

logger = logging.getLogger('CrawlLogger')

FLAGS = gflags.FLAGS

gflags.DEFINE_boolean('new100', False, "crawl new 100")
gflags.DEFINE_boolean('top500', False, "crawl top 500")
gflags.DEFINE_integer('uid', 0, "artist id")
gflags.DEFINE_string('name', '', "artist name")
gflags.DEFINE_boolean('skip', True, "skip exists file")
gflags.DEFINE_string('path', '/Users/chris/mp3', "download path")
#gflags.DEFINE_string('path', '/Volumes/BACKUP/mp3', "download path")

def crawl_item_main():
    if FLAGS.uid:
        artist_url = "http://music.baidu.com/artist/%s" % FLAGS.uid
        crawl_artist(artist_url)
    elif FLAGS.new100:
        crawl_new100()
    elif FLAGS.top500:
        crawl_top500()
    elif FLAGS.name:
        for name in FLAGS.name.strip().split(","):
            try:
                logger.debug("searching %s", name)
                url = "http://music.baidu.com/search?key=%s" % name
                data = download(url)
                logger.debug("parsing search result")
                html = parse_html(data)
                artist_urls = html.xpath("//div[@class='artist-info']/div[@class='name']/a/@href")
                crawl_artist("http://music.baidu.com%s" % artist_urls[0])
            except KeyboardInterrupt:
                raise
            except:
                pass

def crawl_new100():
    url = 'http://music.baidu.com/top/new'
    crawl_top(url)

def crawl_top500():
    url = 'http://music.baidu.com/top/dayhot'
    crawl_top(url)

def crawl_top(url):
    headers = {'User-Agent' : DEFAULT_UA}
    song_headers = {'User-Agent' : DEFAULT_UA, 'Referer' : url}
    logger.debug("downloading %s", url)
    data = download(url, headers)
    logger.debug("parsing %s", url)
    html = parse_html(data)
    part1 = html.xpath("//span[@class='song-title']/a")
    rest = html.xpath("//div[@id='restPage']/textarea/text()")
    #import pdb; pdb.set_trace()
    for p in rest:
        html2 = parse_html(unicode(p))
        part1.extend(html2.xpath("//span[@class='song-title']/a"))
    for song in part1:
        crawl_song(song, "", headers)

def crawl_artist(artist_url):
    headers = {'User-Agent' : DEFAULT_UA}
    song_headers = {'User-Agent' : DEFAULT_UA, 'Referer' : artist_url}
    logger.debug("downloading %s", artist_url)
    data = download(artist_url, headers)
    logger.debug("parsing %s", artist_url)
    html = parse_html(data)
    singer_name = html.xpath("//h2[@class='singer-name']/text()")[0]
    logger.debug("processing song list for %s", singer_name)
    song_list = html.xpath('//span[@class="song-title"]/a')
    for song in song_list:
        crawl_song(song, singer_name, song_headers)
    
def crawl_song(song, singer_name, song_headers):
    #song_title = song.get('title')
    song_title = song.text.strip()
    #import pdb; pdb.set_trace()
    if singer_name:
        file_name = "%s/%s/%s.mp3" % (FLAGS.path, singer_name.replace('(','.').replace(')', '.'), song_title)
    else:
        file_name = "%s/%s.mp3" % (FLAGS.path, song_title)
    logger.debug(file_name)
    if os.path.exists(file_name) and FLAGS.skip:
        logger.info("%s exists, skip", file_name)
        return
    make_dirs_for_file(file_name)
    song_url = "http://music.baidu.com%s/download" % song.get('href')
    logger.debug("\tdownloading %s", song_url)
    song_data = download(song_url , song_headers)
    logger.debug("\tparsing %s", song_url)
    song_html = parse_html(song_data)
    download_urls = song_html.xpath("//div/a[@id='download']/@href")
    if len(download_urls) != 1:
        logger.warn("url error %s", download_urls)
        return
    else:
        logger.debug("\tdownloading mp3 %s", download_urls[0])
        songdata = download("http://music.baidu.com%s" % download_urls[0])
        if not songdata:
            logger.error("\tdownloading failed %s", download_urls[0])
            return
        logger.debug("\twriting to mp3 file %s, len %s", file_name, len(songdata))
        f = open(file_name, "wb")
        f.write(songdata)
        f.close()

if __name__ == "__main__":
    log_init("CrawlLogger", "sqlalchemy.*")
    crawl_item_main()