scrapy 分布式爬虫- RedisSpider

爬去当当书籍信息

多台机器同时爬取，共用一个redis记录 scrapy_redis

带爬取的request对象储存在redis中，每台机器读取request对象并删除记录，经行爬取。实现分布式爬虫

import scrapy

from scrapy_redis.spiders import RedisSpider

from copy import deepcopy

class DangdangSpider(RedisSpider):

    name = 'dangdang'

    allowed_domains = ['dangdang.com']

    # 开始爬虫，会从redis的key中读取start_url.

    redis_key = "dangdang" # lpush dangdang 'http://book.dangdang.com/'

    def parse(self, response):

        # 大分类

        div_list = response.xpath("//div[@class='con flq_body']/div")[:-4]

        print(len(div_list), 'duoshao')

        for div in div_list:

            item = {}

            item['b_cate'] = div.xpath("./dl/dt//text()").extract()

            item['b_cate'] = [i.strip() for i in item['b_cate'] if len(i.strip())>0] # 过滤掉空字符

            print('b_cate:', item['b_cate'])

            # 中间分类

            if item['b_cate'] == ['创意文具']:

                print(item['b_cate'], "pass......")

                item['m_cate'] = None

                item['s_cate_url'] = div.xpath("./dl/dt/a/@ddt-src").extract_first()

                print('s_cate_url:', item['m_cate'])

                # yield scrapy.Request(

                #     item['s_cate_url'],

                #     callback=self.parse_special,

                #     meta={'item': deepcopy(item)}

                # )

            else:

                dl_list = div.xpath(".//dl[@class='inner_dl']")

                for dl in dl_list:

                    item['m_cate'] = dl.xpath("./dt//text()").extract()

                    item['m_cate'] = [i.strip() for i in item['m_cate'] if len(i.strip())>0]

                    # 小分类

                    dd_list = dl.xpath("./dd")

                    for dd in dd_list:

                        item['s_cate'] = dd.xpath("./a/@title").extract_first()

                        item['s_cate_url'] = dd.xpath("./a/@ddt-src").extract_first()

                        # 小分类的所有书籍

                        if item['s_cate_url'] is not None:

                            yield scrapy.Request(

                                item['s_cate_url'],

                                callback=self.parse_books,

                                meta={'item': deepcopy(item)}

                            )

    def parse_special(self, response):

        ''' 文具信息 '''

        pass

    def parse_books(self, response):

        item = response.meta['item']

        # 当前小分类的书籍

        li_list = response.xpath("//ul[@class='list_aa ']/li")

        if li_list is not None:

            for li in li_list:

                try:

                    item['book_price'] = li.xpath(".//span[@class='num']/text()").extract_first() + \

                                         li.xpath(".//span[@class='tail']/text()").extract_first()

                except:

                    item['book_price'] = 'Unknown'

                item['book_url'] = li.xpath("./a/@href").extract_first()

                if item['book_url'] is not None:

                    yield scrapy.Request(

                        item['book_url'],

                        callback=self.parse_book_detail,

                        meta={'item': deepcopy(item)}

                    )

    def parse_book_detail(self, response):

        item = response.meta['item']

        item['book_name'] = response.xpath("//div[@class='name_info']/h1/img/text()").extract_first()

        item['book_desc'] = response.xpath("//span[@class='head_title_name']/text()").extract_first()

        # 这一本书籍的详细信息

        span_list = response.xpath("//div[@class='messbox_info']/span")

        item['book_author'] = span_list.xpath("./span[1]/a/text()").extract() # 可能多个作者

        item['publisher'] = span_list.xpath("./span[2]/a/text()").extract_first()

        item['pub_date'] = span_list.xpath("./span[3]/text()").extract_first()

        print(item)

        # yield item