Python 爬虫练习项目——异步加载爬取

项目代码

from bs4 import BeautifulSoup

import requests

url_prefix = 'https://knewone.com/discover?page='

infos = []

# 获取单个页面数据

def getAPage(url,data = None):

    web_data = requests.get(url)

    soup = BeautifulSoup(web_data.text,'lxml')

    # print(soup)

    images = soup.select('header > a > img')

    titles = soup.select('section > h4 > a')

    links = soup.select('a.cover-inner')

    likes = soup.select('span.fanciers_count')

    if data == None:

        for image,title,link,like in zip(images,titles,links,likes):

            data = {

                'image':image.get('src'),

                'title':title.get_text(),

                'link':'https://knewone.com' + link.get('href'),

                'like':int(like.get_text())

            }

            print(data)

            infos.append(data)

# 获取多个加载的数据

def getMorePages(start,end):

    for url_suffix in range(start,end):

        getAPage(url_prefix + str(url_suffix))

        print('---------------已经获取{}条数据---------------'.format(len(infos)), sep='\n')

# 获取点赞排名前几的数据

def getInfosByLikes(order,infos =infos):

    infos = sorted(infos,key= lambda info:info['like'],reverse = True)

    for info in infos[:order]:

        print(info['like'],info['title'],info['image'],info['link'])

getMorePages(1,4)

getInfosByLikes(5)

项目特点：

【转载】同步加载、异步加载、延迟加载

爬取的网站链接

KnewOne

需要下载代码的可以到我的GitHub上下载 https://github.com/FightingBob/-Web-Crawler-training  如果觉得可以，请给我颗star鼓励一下，谢谢！