Python 多进程爬虫实例

Python  多进程爬虫实例

import json

import re

import time

from multiprocessing import Pool

import requests

from requests.exceptions import RequestException

from bs4 import BeautifulSoup

def get_one_page(url):

    try:

        response = requests.get(url)

        if response.status_code == 200:

            return response.text

        return None

    except RequestException:

        return None

def parse_one_page(html):

    data_list = []

    soup = BeautifulSoup(html, "lxml")

    index_list = soup.select('i.board-index')

    img_list = [x['data-src'] for x in soup.findAll('img', {'class': 'board-img'})]

    name_list = soup.select('p.name')

    actor_list = soup.select('p.star')

    time_list = soup.select('p.releasetime')

    score_list = soup.select('p.score')

    for i in range(len(index_list)):

        data_list.append({

            'index': index_list[i].get_text(),

            'image': img_list[i],

            'title': name_list[i].get_text(),

            'actor': actor_list[i].get_text().strip(),

            'time': time_list[i].get_text(),

            'score': score_list[i].get_text()

        })

    return data_list

def write_to_file(content):

    with open('resul1t.txt', 'a', encoding='utf-8') as f:

        f.write(json.dumps(content, ensure_ascii=False) + '\n')

        f.close()

def main(offset_list):

    for offset in offset_list:

        url = 'http://maoyan.com/board/4?offset=' + str(offset)

        html = get_one_page(url)

        if html:

            for item in parse_one_page(html):

                write_to_file(item)

if __name__ == '__main__':

    # pool = Pool()

    # pool.map(main, [i * 10 for i in range(10)])

    # pool.close()

    # pool.join()

    # main(1)

    offset_list = list(range(0, 100, 10))  # 多进程

    p = Pool()

    for index in range(5):

        p.apply_async(main, args=(offset_list[index * 2:(index + 1) * 2],))

    p.close()

    p.join()