<爬虫实战>糗事百科

1.糗事百科段子.py

# 目标：爬取糗事百科段子信息（文字）

# 信息包括：作者头像，作者名字，作者等级，段子内容，好笑数目，评论数目

# 解析用学过的几种方法都实验一下①正则表达式.②BeautifulSoup③xpath

import requests

import re  # 正则表达式

import json

from bs4 import BeautifulSoup  # BS

from lxml import etree  # xpath

def get_one_page(url):

	response = requests.get(url)

	if response.status_code == 200:

		return response.text

	return None

def zhengze_parse(html):

	pattern = re.compile(

		'<img src="//(.*?)".*?alt="(.*?)".*?<a.*?<div class=".*?">(.*?)</div>'

		+ '.*?<div class=.*?<span>(.*?)</span>.*?<span class=".*?".*?<i.*?>(.*?)<'

		+ '.*?<i.*?>(.*?)<',

		re.S)

	items = re.findall(pattern, html)

	for item in items:

		content = item[3].replace('<br/>', '').strip()

		content = content.replace('\x01', '')

		if item[5] == '京公网安备11010502031601号':

			break

		yield {

			'image': "http://" + item[0],

			'name': item[1],

			'grade': item[2],

			'content': content,

			'fun_Num': item[4],

			'com_Num': item[5]

		}

def soup_parse(html):

	soup = BeautifulSoup(html, 'lxml')

	for data in soup.find_all('div', class_='article'):

		image = "http:" + data.img['src']

		name = data.img['alt']

		# 匿名用户没有等级

		if name=="匿名用户":

			grade = "匿名用户"

		else:

			grade = data.find('div', class_='articleGender').text

		content = data.find('div', class_='content').span.text.strip()

		fun_Num = data.find('i', class_='number').text

		com_Num = data.find('a', class_='qiushi_comments').i.text

		yield {

			'image': image,

			'name': name,

			'grade': grade,

			'content': content,

			'fun_Num': fun_Num,

			'com_Num': com_Num,

		}

def xpath_parse(html):

	html = etree.HTML(html)

	for data in html.xpath('//div[@class="col1"]/div'):

		image = "http:"+ str(data.xpath('.//img/@src')[0])

		name = data.xpath('.//img/@alt')[0]

		if name == '匿名用户':

			grade = '匿名用户'

		else:

			grade = data.xpath('./div[1]/div/text()')[0]

		content = data.xpath('./a/div/span/text()')[0:]

		content = str(content).strip().replace('\\n','')

		fun_Num = data.xpath('./div[2]/span[1]/i/text()')[0]

		com_Num = data.xpath('.//div[2]/span[2]/a/i/text()')[0]

		# print(image, name, grade, content, fun_Num, com_Num)

		yield {

			'image': image,

			'name': name,

			'grade': grade,

			'content': content,

			'fun_Num': fun_Num,

			'com_Num': com_Num,

		}

def write_to_file(content, flag):

	with open('糗百段子(' + str(flag) + ').txt', 'a', encoding='utf-8')as f:

		f.write(json.dumps(content, ensure_ascii=False) + '\n')

def search(Num):

	url = 'https://www.qiushibaike.com/text/page/' + str(Num) + '/'

	html = get_one_page(url)

	# 正则匹配不到匿名用户的等级，不会匹配匿名用户的段子，所以少一些数据

	# 稍微加个判断逻辑就行了，懒得弄了

	for item in zhengze_parse(html):

		write_to_file(item, '正则表达式')

	for item in soup_parse(html):

		write_to_file(item, 'BS4')

	for item in xpath_parse(html):

		write_to_file(item, 'xpath')

	page = str(Num)

	print("正在爬取第" + page + '页')

def main():

	# 提供页码

	for Num in range(1, 14):

		search(Num)

	print("爬取完成")

if __name__ == '__main__':

	# 入口

	main()

　　

2.打包

pyinstaller -F 糗事百科段子.py

　　

3.运行效果

网页上匿名用户段子的显示情况