抓包时发现子菜单请求数据时一般需要rid,但的确存在一些如游戏->游戏赛事不使用rid,对于这种未进行处理,此外rid一般在主菜单的响应中,但有的如番剧这种,rid在子菜单的url中,此外返回的data中含有页数相关信息,可以据此定义爬取的页面数量
# -*- coding: utf-8 -*-
# @author: Tele
# @Time : 2019/04/08 下午 1:01
import requests
import json
import os
import re
import shutil
from lxml import etree # 爬取每个菜单的前5页内容
class BiliSplider:
def __init__(self, save_dir, menu_list):
self.target = menu_list
self.url_temp = "https://www.bilibili.com/"
self.headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
# "Cookie": "LIVE_BUVID=AUTO6715546997211617; buvid3=07192BD6-2288-4BA5-9259-8E0BF6381C9347193infoc; stardustvideo=1; CURRENT_FNVAL=16; sid=l0fnfa5e; rpdid=bfAHHkDF:cq6flbmZ:Ohzhw:1Hdog8",
}
self.proxies = {
"http": "http://61.190.102.50:15845"
}
self.father_dir = save_dir def get_menu_list(self):
regex = re.compile("//")
response = requests.get(self.url_temp, headers=self.headers)
html_element = etree.HTML(response.content)
nav_menu_list = html_element.xpath("//div[@id='primary_menu']/ul[@class='nav-menu']/li/a") menu_list = list()
for item in nav_menu_list:
menu = dict()
title = item.xpath("./*/text()")
menu["title"] = title[0] if len(title) > 0 else None
href = item.xpath("./@href")
menu["href"] = "https://" + regex.sub("", href[0]) if len(href) > 0 else None # 子菜单
submenu_list = list()
sub_nav_list = item.xpath("./../ul[@class='sub-nav']/li")
if len(sub_nav_list) > 0:
for sub in sub_nav_list:
submenu = dict()
sub_title = sub.xpath("./a/span/text()")
submenu["title"] = sub_title[0] if len(sub_title) > 0 else None
sub_href = sub.xpath("./a/@href")
submenu["href"] = "https://" + regex.sub("", sub_href[0]) if len(sub_href) > 0 else None
submenu_list.append(submenu)
menu["submenu_list"] = submenu_list if len(submenu_list) > 0 else None
menu_list.append(menu)
return menu_list # rid=tid
def parse_index_url(self, url):
result_list = list()
# 正则匹配
regex = re.compile("<script>window.__INITIAL_STATE__=(.*)</script>")
response = requests.get(url, headers=self.headers)
result = regex.findall(response.content.decode())
temp = re.compile("(.*);\(function").findall(result[0]) if len(result) > 0 else None
sub_list = json.loads(temp[0])["config"]["sub"] if temp else list()
if len(sub_list) > 0:
for sub in sub_list:
# 一些子菜单没有rid,需要请求不同的url,暂不处理
if "tid" in sub:
if sub["tid"]:
sub_menu = dict()
sub_menu["rid"] = sub["tid"] if sub["tid"] else None
sub_menu["title"] = sub["name"] if sub["name"] else None
result_list.append(sub_menu)
else:
pass return result_list # 最新动态 region?callback
# 数据 newlist?callback
def parse_sub_url(self, item):
self.headers["Referer"] = item["referer"]
url_pattern = "https://api.bilibili.com/x/web-interface/newlist?rid={}&type=0&pn={}&ps=20" # 每个菜单爬取前5页
for i in range(1, 6):
data = dict()
url = url_pattern.format(item["rid"], i)
print(url)
try:
response = requests.get(url, headers=self.headers, proxies=self.proxies, timeout=10)
except:
return
if response.status_code == 200:
data["content"] = json.loads(response.content.decode())["data"]
data["title"] = item["title"]
data["index"] = i
data["menu"] = item["menu"]
# 保存数据
self.save_data(data)
else:
print("请求超时") # 一般是403,被封IP了 def save_data(self, data):
if len(data["content"]) == 0:
return
parent_path = self.father_dir + "/" + data["menu"] + "/" + data["title"]
if not os.path.exists(parent_path):
os.makedirs(parent_path)
file_dir = parent_path + "/" + "第" + str(data["index"]) + "页.txt" # 保存
with open(file_dir, "w", encoding="utf-8") as file:
file.write(json.dumps(data["content"], ensure_ascii=False, indent=2)) def run(self):
# 清除之前保存的数据
if os.path.exists(self.father_dir):
shutil.rmtree(self.father_dir) menu_list = self.get_menu_list()
menu_info = list()
# 获得目标菜单信息
# 特殊列表,一些菜单的rid必须从子菜单的url中获得
special_list = list()
for menu in menu_list:
for t in self.target:
if menu["title"] == t:
if menu["title"] == "番剧" or menu["title"] == "国创" or menu["title"] == "影视":
special_list.append(menu)
menu_info.append(menu)
break # 目标菜单的主页
if len(menu_info) > 0:
for info in menu_info:
menu_index_url = info["href"]
# 处理特殊列表
if info in special_list:
menu_index_url = info["submenu_list"][0]["href"]
# 获得rid
result_list = self.parse_index_url(menu_index_url)
print(result_list)
if len(result_list) > 0:
for item in result_list:
# 大菜单
item["menu"] = info["title"]
item["referer"] = menu_index_url
# 爬取子菜单
self.parse_sub_url(item) def main():
target = ["动画", "番剧", "国创", "音乐", "舞蹈", "游戏", "科技", "数码", "生活", "鬼畜", "时尚", "广告", "娱乐", "影视"]
splider = BiliSplider("f:/bili_splider", target)
splider.run() if __name__ == '__main__':
main()
可以看到番剧少了新番时间表与番剧索引,因为这两个请求不遵循https://api.bilibili.com/x/web-interface/newlist?rid={}&type=0&pn={}&ps=20的格式,类似的不再赘述