用python提取百度贴吧的小说

#-*- encoding: utf-8 -*-
import urllib2
import re
from BeautifulSoup import BeautifulSoup

def stripHTMLTags (html):
    '''strip html tags; from http://goo.gl/EaYp5'''
    return re.sub('<([^!>]([^>]|\n)*)>', '', html)

def fetch_tieba(url,localfile,ignoreFansReq=False):
    '''fetch the url resource and save to localfile'''
    
    # fetch the url resource and encode to utf-8
    html = urllib2.urlopen(url).read()
    html = unicode(html,'gb2312','ignore').encode('utf-8','ignore')

    # extract the main content
    content = BeautifulSoup(html).findAll(attrs={'class':'d_post_content'})

    # write the content to localfile
    myfile = open(localfile,'w')
    for item in content:
        item_formatted = stripHTMLTags(str(item).replace('<br />','\r\n'))
        if ignoreFansReq == True :
            if len(item_formatted) < 100:
                continue
        myfile.write(item_formatted)
        myfile.write('\r\n')
        print item_formatted
    myfile.close()

def main():
    urlTarget = "http://tieba.baidu.com/p/1234371208"
    localfileTarget = './xiaoshuo2.txt'
    fetch_tieba(url=urlTarget,localfile=localfileTarget,ignoreFansReq=True)

if __name__ == "__main__":
    main()