scrapy(四): 爬取二级页面的内容

scrapy爬取二级页面的内容

1.定义数据结构item.py文件

# -*- coding: utf-8 -*-

'''

field: item.py

'''

# Define here the models for your scraped items

#

# See documentation in:

# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class TupianprojectItem(scrapy.Item):

    # define the fields for your item here like:

    # name = scrapy.Field()

    # 图片标题

    title = scrapy.Field()

    # 发布时间

    publish_time = scrapy.Field()

    # 浏览量

    look = scrapy.Field()

    # 收藏量

    collect = scrapy.Field()

    # 下载量

    download = scrapy.Field()

    # 图片链接

    image_url = scrapy.Field()

2.爬虫文件

# -*- coding: utf-8 -*-

import scrapy

from tupianproject.items import TupianprojectItem

class ImageSpider(scrapy.Spider):

    name = 'image'

    allowed_domains = ['699pic.com']

    start_urls = ['http://699pic.com/people-1-0-0-0-0-0-0.html']

    url = 'http://699pic.com/people-{}-0-0-0-0-0-0.html'

    page = 1

    def parse(self, response):

        # 在一级页面中，应该将所有的图片详情页面的链接获取到

        image_detail_url_list = response.xpath('//div[@class="list"]/a/@href').extract()

        # pass

        # 遍历详情页面，向每一个详情页面发送请求即可

        for image_detail_url in image_detail_url_list:

            yield scrapy.Request(url=image_detail_url, callback=self.parse_detail)

        # 接着发送其他请求

        if self.page <= 3:

            self.page += 1

            url = self.url.format(self.page)

            yield scrapy.Request(url=url, callback=self.parse)

    def parse_detail(self, response):

        # 创建一个item对象

        item = TupianprojectItem()

        # 提取图片的每一个信息

        # title

        item['title'] = response.xpath('//div[@class="photo-view"]/h1/text()').extract_first()

        # 发布时间

        item['publish_time'] = response.xpath('//div[@class="photo-view"]/div/span[@class="publicityt"]')[0].xpath('string(.)').extract_first()

        # 获取浏览量

        item['look'] = response.xpath('//div[@class="photo-view"]/div/span[@class="look"]/read/text()').extract_first()

        # 获取收藏量

        item['collect'] = response.xpath('//div[@class="photo-view"]/div/span[@class="collect"]')[0].xpath('string(.)').extract_first()

        # 获取下载量

        item['download'] = response.xpath('//div[@class="photo-view"]/div/span[@class="download"]')[0].xpath('string(.)').extract_first().strip('\n\t')

        # 获取图片的链接

        item['image_url'] = response.xpath('//div[@class="huabu"]//img/@src').extract_first()

        # 将item发送出去

        yield item

3.管道文件

# -*- coding: utf-8 -*-

'''

filed: pipelines.py

'''

s

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

import json

import urllib.request

import os

class TupianprojectPipeline(object):

    def open_spider(self, spider):

        self.fp = open('tupian.json', 'w', encoding='utf8')

    def process_item(self, item, spider):

        d = dict(item)

        string = json.dumps(d, ensure_ascii=False)

        self.fp.write(string + '\n')

        # 下载图片

        self.download(item)

        return item

    def download(self, item):

        dirname = './people'

        suffix = item['image_url'].split('.')[-1]

        filename = item['title'] + '.' + suffix

        filepath = os.path.join(dirname, filename)

        urllib.request.urlretrieve(item['image_url'], filepath)

    def close_spider(self, spider):

        self.fp.close()