爬虫实例

1. 爬校花网图片：

这是一个很基础的爬虫小例子，爬取校花网的图片。其中用了requests-html库：

先获取每页的url，再爬取每页中的图片进行解析处理，最后存入到文件中

from requests_html import HTMLSession
import os

class Spider():
    def __init__(self):
        self.session = HTMLSession()

    def get_index_page(self):
        for i in range(1, 4):
            if i == 1:
                page_url = "http://www.xiaohuar.com/meinv/index.html"
            else:
                page_url = "http://www.xiaohuar.com/meinv/index_%s.html" % i
            yield page_url

    def get_image_name(self, page_url):
        r = self.session.get(url=page_url)
        elements_list = r.html.find('#images .items')
        for element in elements_list:
            image_url = element.find('img', first=True).attrs.get('src')
            image_name = element.find('.p_title a', first=True).text
            image_name = image_name.replace('【', '').replace('】', '').replace('|', '').replace('\\', '').replace(
                '/','') + '.jpg'
            yield image_url, image_name

    def save(self, image_url, image_name):
        image_name = os.path.join('pictures', image_name)
        if not image_url.startswith('http'):
            image_url = 'http://www.xiaohuar.com' + image_url
        r = self.session.get(url=image_url)
        with open(image_name, 'wb') as f:
            f.write(r.content)
            print('%s下载完成' % image_name)

    def run(self):
        for page_url in self.get_index_page():
            for image_url, image_name in self.get_image_name(page_url):
                self.save(image_url, image_name)


if __name__ == '__main__':
    xiaohua = Spider()
    xiaohua.run()

2. 豆瓣电影排行信息

爬虫获取豆瓣电影信息：在这里对电影进行筛选是通过url携带参数。所以先获取参数信息。

from requests_html import HTMLSession

class Spider:
    def __init__(self):
        self.api = "https://movie.douban.com/j/new_search_subjects?"
        self.session = HTMLSession()

    def get_params(self):
        sort = input('请输入按什么排序（S评分）')
        year_range = input('请输入年份：')
        self.params = {
            'sort':sort,
            'year_range':year_range,
            'start':0
        }


    def get_message(self):
        for i in range(10):
            self.params['start'] = i * 20
            r = self.session.get(url=self.api,params=self.params)
            print(r.json())

    def run(self):
        self.get_params()
        self.get_message()
if __name__ == '__main__':
    douban = Spider()
    douban.run()

3. 爬取校花视频

校花视频是通过m3u8格式。有的视频会员有反爬机制，查看元素的播放链接是unknown，所以获取不到资源。

现获取到m3u8格式的播放列表连接，然后发送请求获取到的文件内容是一行行.ts。.ts也是一个文件格式。对m3u8文件内容就行处理，发送请求下载.ts文件保存

from requests_html import HTMLSession
import os

class spider():

    def __init__(self):
        self.session = HTMLSession()

    def get_index_page(self):
        for i in range(7):
            url = 'http://www.xiaohuar.com/list-3-%s.html'%i
            yield url

    def parse_index_page(self,index_page):
        r = self.session.get(url=index_page)
        elements_list = r.html.find('#images .items a[class="imglink"]')
        for element in elements_list:
            yield element.attrs.get('href')

    def parse_detail_page(self,detail_page):
        r = self.session.get(url=detail_page)
        r.html.encoding = 'GBK'
        result_obj = r.html.search('var vHLSurl    = "{}";')
        if result_obj:
            m3u8_url = result_obj[0]
            m3u8_name = r.html.find('title',first=True).text.replace('\\','')
            yield m3u8_url,m3u8_name
        else:
            print("匹配失败，无资源")

    def save_m3u8(self,m3u8_url,m3u8_name):
        m3u8_dir = m3u8_name
        if not os.path.exists(m3u8_dir):
            os.mkdir(m3u8_dir)
        print(m3u8_url)
        r = self.session.get(url=m3u8_url)
        m3u8_path = os.path.join(m3u8_dir,'playlist.m3u8')
        with open(m3u8_path,'wt+',encoding='utf-8') as f :
            f.write(r.text)
            f.seek(0,0)
            for line in f:
                line = line.strip()
                if line.endswith('.ts'):
                    ts_url = os.path.dirname(m3u8_url) + '/%s'%line
                    r = self.session.get(url=ts_url)
                    ts_path =  os.path.join(m3u8_dir,line)
                    with open(ts_path,'wb') as f1:
                        f1.write(r.content)
                        print('%s下载完毕'%line)

    def run(self):
        for url in self.get_index_page():
            for detail_page in  self.parse_index_page(url):
                for m3u8_url,m3u8_name in self.parse_detail_page(detail_page):
                    self.save_m3u8(m3u8_url,m3u8_name)


if __name__ == '__main__':
    xioahua = spider()
    xioahua.run()