aiohttp使用队列

获取百度的搜索结果,然后把百度的长链接,获取到真实的url

import time

import aiofiles

import aiohttp

import asyncio

from lxml import etree

from asyncio import Queue

from itertools import product

import async_timeout

MAX_THREADS = 50

class BaiduSpider:

    def __init__(self):

        self.headers = {

            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36"

                          "(KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"}

        self.q = Queue()

        self.q2 = Queue()

    def url_generator(self):

        with open('keyword.txt', 'r', encoding='utf8') as f:

            for key in product(f, range(0, 5)):

                yield f"https://www.baidu.com/s?wd={key[0].strip()}&pn={key[1]}"

    async def fetch(self, session, url):

        try:

            with async_timeout.timeout(1):

                async with session.get(url, headers=self.headers) as resp:

                    if resp.status in [200, 201]:

                        return await resp.text()

        except Exception as e:

            pass

    async def work(self, session):

        while not self.q.empty():

            url = await self.q.get()

            html = await self.fetch(session, url)

            datas = await self.parser(session, html)

            self.q.task_done()

    async def parser(self, session, html):

        if html:

            tree = etree.HTML(html)

            datas = tree.xpath('//h3[@class="t"]/a')

            for data in datas:

                title = data.xpath('string(.)')

                link = data.xpath('@href')[0]

                data = [title, link if title else '']

                self.q2.put_nowait(data)

            await self.work2(session)

    async def work2(self, session):

        while not self.q2.empty():

            data = await self.q2.get()

            try:

                with async_timeout.timeout(1):

                    async with session.get(data[1], headers=self.headers) as resp2:

                        print(resp2.url, data[0])

                        async with aiofiles.open('links.txt', 'a', encoding='utf-8') as fd:

                            if str(resp2.url) not in 'links.txt':

                                await fd.write(f"{data[0]},{resp2.url}\n")

            except Exception as e:

                pass

    async def download(self):

        urls = self.url_generator()

        conn = aiohttp.TCPConnector(verify_ssl=False)  # 防止ssl报错

        [self.q.put_nowait(url) for url in urls]

        async with aiohttp.ClientSession(connector=conn) as session:

            tasks = [asyncio.ensure_future(self.work(session)) for _ in range(MAX_THREADS)]

            await asyncio.wait(tasks)

    def run(self):

        start_time = time.time()

        loop = asyncio.get_event_loop()

        tasks1 = asyncio.gather(self.download())

        loop.run_until_complete(tasks1)

        print(f'全程用时{time.time() - start_time}秒')

if __name__ == '__main__':

    baidu = BaiduSpider()

    items = baidu.run()