Python3 多线程爬取梨视频

多线程爬取梨视频

from threading import Thread

import requests

import re

# 访问链接

def access_page(url):

    response = requests.get(url)

    return response

# 获取主页视频的id列表, 用来拼接视频详情页链接

def get_video_id(homepage_data):

    id_list = re.findall('<a href="video_(.*?)" .*?>', homepage_data, re.S)

    return id_list

# 获取视频链接列表

def get_video_url(detail_page_data):

    video_url = re.findall('srcUrl="(.*?)"', detail_page_data, re.S)[0]

    return video_url

# 获取视频名称

def get_video_name(detail_page_date):

    video_name = re.findall('<h1 class="video-tt">(.*?)</h1>', detail_page_date, re.S)[0]

    # 替换其中的非法字符, 不然在作为文件名保存时候会报错

    remove_str = r'[\/\\\:\*\?\"\<\>\|]'

    video_name = re.sub(remove_str, '_', video_name)

    return video_name

# 保存视频

def save(video_data, name):

    with open(f'{name}.mp4', 'wb') as f:

        f.write(video_data)

        print(f'视频[{name}]下载成功!')

def run(id):

    # 拼接详情页链接并访问

    detail_page_url = 'https://www.pearvideo.com/video_' + id

    detail_page_data = access_page(detail_page_url).text

    # 获取视频名称和视频链接

    video_name = get_video_name(detail_page_data)

    video_url = get_video_url(detail_page_data)

    # 访问视频链接获取视频数据

    video_data = access_page(video_url).content

    # 保存视频数据

    save(video_data, video_name)

if __name__ == '__main__':

    homepage_data = access_page('https://www.pearvideo.com/').text

    id_list = get_video_id(homepage_data)

    # 多线程爬取

    for id in id_list:

        t = Thread(target=run, args=(id,))

        t.start()