爬虫_豆瓣电影top250 （正则表达式）

一样的套路，就是多线程还没弄

 import requests

 import re

 import json

 headers = 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'

 def get_one_page(url):

     try:

         html = requests.get(url, headers={'User-Agent':'headers'})

         if html.status_code == 200:

             return html.text

         return None

     except RequestsException:

         return None

 def write_txt(content):

     with open('result.txt', 'a', encoding='utf-8') as f:

         f.write(json.dumps(content, ensure_ascii=False) + '\n')

         f.close

 def parse_one_page(html):

     # <em class="">(\d+)</em>

     # .*?href="(.*?)/">.*?

     # other">(\w+)</span

     match = re.compile('.*?<em class="">(.*?)</em>.*?href="(.*?)/">.*?"title">(.*?)</span.*?other">(.*?)</span', re.S)

     results = re.findall(match, html)

     for item in results:

         yield{

             'range': item[0],

             'movie_main_page': item[1],

             'movie_title': item[2],

             'other_name': item[3].strip()[13:]

         }

     # print(results)

 def main():

     for start in range(0, 250, 25):

         url = 'https://movie.douban.com/top250?start=' + str(start)

         html = get_one_page(url)

         for item in parse_one_page(html):

             print(item)

             write_txt(item)

 if __name__ == '__main__':

     main()

运行结果