aiohttp的模板

 import aiohttp

 import asyncio

 import async_timeout

 from urllib.parse import urljoin,urldefrag

 root_url = 'http://python/org/'  # 开始的url

 crawled_urls,url_hub = [], [root_url]

 headers = {'user-agent': 'Opera/9.80 (X11; Linux x86_64; U; en) Presto/2.2.15 Version/10.10'}

 async def get_body(url):

     async with aiohttp.ClientSession() as session:

         try:

             with async_timeout.timeout(10): #超时时间的设定

                 async with session.get(url,headers=headers) as response:

                     if response.status == 200:

                         html = await response.text()

                         return {'error':'','html':html,'url':url}

                     else:

                         return {'error':response.status,'html':'','url':url}

         except Exception as err:

             return {'error':response.status,'html':'','url':url}

 async def handle_task(task_id,work_queue):

     while not work_queue.empty():

         queue_url = await work_queue.get()

         if not queue_url in crawled_urls:

             body = await get_body(queue_url)

             if not body['error']:

                 crawled_urls.append(queue_url)

                 parse(body)

             else:

                 print('{}爬取失败'.format(queue_url))

 #解析返回的数据

 def parse(body):

     pass

 def remove_fragment(url):

     pure_url, frag = urldefrag(url)

     return pure_url

 #解析html，拼接新的url

 def get_urls(html):

     new_urls = [url.split('"')[0] for url in str(html).replace("'", '"').split('href="')[1:]]

     return [urljoin(root_url, remove_fragment(new_url)) for new_url in new_urls]

 if __name__ == '__main__':

     q = asyncio.Queue()  #初始化一个异步的队列

     [q.put_nowait(url) for url in url_hub]  #从初始的url队列中遍历，把url放入到队列中

     loop = asyncio.get_event_loop()

     tasks = [handle_task(task_id, q) for task_id in range(3)]  #3个并发

     loop.run_until_complete(asyncio.wait(tasks))

     loop.close()

     for u in crawled_urls:

         print(u)

     print('-' * 30)

     print(len(crawled_urls))