Python爬虫(一)爬百度贴吧

简单的GET请求:

# python2

import urllib2

response = urllib2.urlopen('http://www.baidu.com')

html = response.read()

print html

req = urllib2.Request('http://www.baidu.com')

req.add_header('User-Agent', 'Chrome')

response = urllib2.urlopen(req)

print 'response headers:'

print response.info()

爬百度贴吧一个帖子：

# python2

# -*- coding: utf-8 -*-

import urllib2

import string

def crawl_tieba(base_url, begin_page, end_page):

    for i in range(begin_page, end_page + 1):

        print '正在下载第' + str(i) + '个网页...'

        url = base_url + '?pn=' + str(i)

        m = urllib2.urlopen(url).read()

        file_name = string.zfill(i, 5) + '.html'

        f = open(file_name, 'w+')

        f.write(m)

        f.close()

    print 'done'

crawl_tieba('http://tieba.baidu.com/p/4999189637', 1, 10)

WARNING:

　　如果没有第二行的注释，会报错：" SyntaxError: Non-ASCII character '\xe6' "。

爬糗百的帖子：

# python2

# -*- coding: utf-8 -*-

import urllib2

import string

import re

def crawl_qiubai(base_url, begin_page, end_page):

    for i in range(begin_page, end_page + 1):

        url = base_url + str(i)

        user_agent = 'chrome'

        headers = {'User-Agent': user_agent}

        print '正在下载网页' + str(i) + '...'

        req = urllib2.Request(url, headers=headers)

        response = urllib2.urlopen(req).read()

        pattern = re.compile(r'<div.*?class="content">.*?<span>(.*?)</span>', re.S)

        features = re.findall(pattern, response)

        file_name = 'qiubai_' + string.zfill(i, 5) + '.txt'

        f = open(file_name, 'w+')

        for index in range(len(features)):

            feature = features[index].replace('<br/>', '\n')

            f.write('第' + str(index + 1) + '条:\n')

            f.write(feature + '\n\n')

        f.close()

        print '网页' + str(i) + '下载完成'

    print 'done'

crawl_qiubai('http://www.qiushibaike.com/hot/page/', 1, 10)

参考资料:

Python爬虫入门教程

谈谈 Python 中的连接符（+、+=）