#calibre fetch cnblogs article

gistfile1.py

import string, re, time
strftime = time.strftime
class MVCInDetail(BasicNewsRecipe):
    title = u'cnblogs test fetch'
    delay = 5
    recursions = 0
#    auto_cleanup = True

    no_stylesheets = True
#keep_only_tags = [dict(id='mainContent')]
    keep_only_tags = [dict(id='cnblogs_post_body')]
#remove_tags = [dict(name='div', attrs={'class':'footer'})]

    def parse_index(self):
        soup = self.index_to_soup('http://www.cnblogs.com/who/you_want_to_fetch')

        def feed_title(div):
            return ''.join(div.findAll(text=True, recursive=False)).strip()

        articles = {}
        key = None
        ans = []
        for book in soup.findAll(True,
                             attrs={'id':'cnblogs_post_body'}):            
            for div in book.findAll():
                if div.name == 'p':
                    key = string.capwords(feed_title(div))
                    articles[key] = []
                    ans.append(key)


                elif div.name == 'blockquote':
                    links=div.findAll('a')
                    for a in links:
                        if not a:
                            continue
                        url = re.sub(r'\?.*', '', a['href'])
                        title = self.tag_to_string(a, use_alt=True).strip()
                        description = ''
                        pubdate = strftime('%a, %d %b')
                        summary = div.find(True, attrs={'class':'summary'})
                        if summary:
                            description = self.tag_to_string(summary, use_alt=False)

                        feed = key if key is not None else 'Uncategorized'
                        if not articles.has_key(feed):
                            articles[feed] = []
                        articles[feed].append(dict(title=title, url=url, date=pubdate,
                                                       description=description,
                                                       content=''))

        ans = [(key, articles[key]) for key in ans if articles.has_key(key)]
        return ans


#    def preprocess_html(self, soup):