python抓取头条文章

python抓取头条美文并存储到mongodb

# Author:song

from multiprocessing import Pool

from urllib.parse import urlencode

import requests

import json

from requests import RequestException

from bs4 import BeautifulSoup

import re

import pymongo

client = pymongo.MongoClient('localhost',connect=False)

db = client['toutiaowenzhang']

def get_index(offset):

    data = {

        'offset': offset,

        'format': 'json',

        'keyword': '美文',

        'autoload': 'true',

        'count': 20,

        'cur_tab': 1,

        'from':'search_tab'

    }

    url = 'https://www.toutiao.com/search_content/?'+urlencode(data)

    response = requests.get(url)

    try:

        if response.status_code == 200:

            return response.text

        else:

            return None

    except RequestException:

        return None

def get_urls(html):

    data = json.loads(html)

    if data and 'data' in data.keys():

        for item in data.get('data'):

            yield item.get('article_url')

def get_index_detail(url):

    response = requests.get(url)

    try:

        if response.status_code == 200:

            return response.text

        else:

            return None

    except RequestException:

        return None

def parse_detail(html):

    try:

        soup = BeautifulSoup(html,'lxml')

        title = soup.select('title')[0].get_text()

        compile_allarticle= re.compile('content.*?<div&gt(.*?)</div>',re.S)

        allarticle = re.findall(compile_allarticle,html)

        # article =re.sub('(<.*?<span>)','',allarticle[0])#正则匹配上不需要的那部分

        article =re.sub('[a-zA-Z0-9/#;&\._]','',str(allarticle)).strip()#直接把字母数字全部替换

        data = {

            'title':title,

            'article':article

        }

        return data

    except TypeError:#解决出现了404界面

        pass

def save_to_mongodb(result):

    if db['toutiaowenzhang'].insert(result):

        print('successful')

    else:

        print('fail')

def main(offset):

    html = get_index(offset)

    items = get_urls(html)

    for item in items:

        if item:

            ab = get_index_detail(item)

            result = parse_detail(ab)

            save_to_mongodb(result)

if __name__=='__main__':

    groups = [x*20 for x in range(3)]

    pool = Pool()

    pool.map(main,groups)