一、items保存爬取的文件

items.py

import scrapy

class QuoteItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
text = scrapy.Field()
author = scrapy.Field()
tags = scrapy.Field()

quote.py

# -*- coding: utf-8 -*-
import scrapy
from toscrapy.items import QuoteItem class QuoteSpider(scrapy.Spider):
name = 'quote'
allowed_domains = ['quotes.toscrape.com']
start_urls = ['http://quotes.toscrape.com/']
"""
知识点
1. text()获取标签的text
2. @属性 获取属性的值
3. extract()查找多个 extract_first() 查找一个
4. response.urljoin url拼接
5. scrapy.Request(url=_next, callback=self.parse) 回调
"""
def parse(self, response):
# print(response.text)
quotes = response.xpath('//div[@class="col-md-8"]/div[@class="quote"]')
# print(quotes)''
for quote in quotes:
# print('=' * 20)
# print(quote)
item = QuoteItem()
# extract_first() 查找一个
text = quote.xpath('.//span[@class="text"]/text()').extract_first()
# print(text)
item['text'] = text
author = quote.xpath('.//span/small[@class="author"]/text()').extract_first()
# print(author)
item['author'] = author
# extract()查找多个
tags = quote.xpath('.//div[@class="tags"]/a[@class="tag"]/@href').extract()
item['tags'] = tags
# print(tags)
yield item
# print('>' * 40)
next_url = response.xpath('//div[@class="col-md-8"]/nav/ul[@class="pager"]/li[@class="next"]/a/@href').extract_first()
# print(next_url)
# 拼接url
_next = response.urljoin(next_url)
# print(_next)
# callback 回调函数
yield scrapy.Request(url=_next, callback=self.parse)

或直接yield QuoteItem()

产生文件命令

scrapy crawl quote -o qutoes.json
scrapy crawl quote -o quotes.jsonlines
or
scrapy crawl quote -o quotes.jl
# 每一个item输出一行json

文件类型:qutoes.xml  qutoes.jl  qutoes.csv等

二、piplines

1、核心:

爬虫每执行一次 yield item对象 -> 执行一次pipelines中的process_item方法(通过修改配置文件使pipelines生效) -> 将数据存入数据库或写入文件

2、settings.py

ITEM_PIPELINES = {
# 后面的参数是不同pipelines类的权重值(0-1000),权重值越小越优先
'toscrapy.pipelines.ToscrapyPipeline': 300,
}

配置settings文件是执行pipeline的前提条件

3、pipelines

a、默认

class ToscrapyPipeline(object):

    def process_item(self, item, spider):
"""
:param item: item对象
:param spider: 爬虫对象
:return:
"""
# print('='*20, item)
return item

b、其它方法

开始爬虫时,调用的方法

    def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
pass

爬虫结束时,调用的方法

    def close_spider(self, spider):
"""
关闭爬虫,调用
:param spider:
:return:
"""
pass

from_crawler方法

作用:初始化时,实例化pipleline类对象

目的:将数据储存的路径,写到配置文件中

    @classmethod
def from_crawler(cls, crawler):
"""
初始化方法时,创建pipeline类的对象
:param crawler:
:return:
"""
# crawler.settings 获取全部配置文件
path = crawler.settings.get('FILE_PATH')
# 实例化对象
return cls(path)

c、pipeline类方法分析

判断是否有from_crawler方法

有:obj = pipeline类.from_crawler()

无:obj = pipeline类()

当爬虫开始时,执行 open_spider方法

当爬虫yield item对象时, 执行 process_item方法

当爬虫结束时,执行 close_spider方法

d、序列化

pipelines.py

class ToscrapyPipeline(object):

    def __init__(self, path):
self.f = None
self.path = path @classmethod
def from_crawler(cls, crawler):
"""
初始化方法时,创建pipeline类的对象
:param crawler:
:return:
"""
# crawler.settings 获取全部配置文件
path = crawler.settings.get('FILE_PATH')
# 实例化对象
return cls(path) def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
self.f = open(file=self.path, mode='a', encoding="utf-8") def process_item(self, item, spider):
"""
爬虫执行yield item对象,调用
:param item: item对象
:param spider: 爬虫对象
:return:
"""
# print('='*20, item)
self.f.write(item['text'] + '\n')
return item def close_spider(self, spider):
"""
爬虫结束,调用
:param spider:
:return:
"""
self.f.close()

4、多个pipeline类

piplines.py文件中可以有多个类,一个把数据保存到数据库,一个把数据保存到文件

a、执行顺序由settings.py的权重值决定,多个pipeline类中方法的执行顺序可以看成有序的异步

file from_crawl
db from_crawl
file open_spider
db open_spider
file process_item
db process_item
file process_item
db process_item
file process_item
db process_item
db close_spider
file close_spider

b、process_item方法中return item的作用

为下一个pipeline类中的process_item方法提供item

1)、没有返回item

下一个类的item是None

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html class ToscrapyPipeline(object): def __init__(self, path):
self.f = None
self.path = path @classmethod
def from_crawler(cls, crawler):
"""
初始化方法时,创建pipeline类的对象
:param crawler:
:return:
"""
# crawler.settings 获取全部配置文件
path = crawler.settings.get('FILE_PATH')
# 实例化对象
print('file from_crawl')
return cls(path) def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
print('file open_spider')
self.f = open(file=self.path, mode='a', encoding="utf-8") def process_item(self, item, spider):
"""
爬虫执行yield item对象,调用
:param item: item对象
:param spider: 爬虫对象
:return:
"""
print('file process_item')
# self.f.write(item['text'] + '\n')
# return item def close_spider(self, spider):
"""
爬虫结束,调用
:param spider:
:return:
"""
print('file close_spider')
self.f.close() class DBPipeline(object): def __init__(self, path):
self.f = None
self.path = path @classmethod
def from_crawler(cls, crawler):
"""
初始化方法时,创建pipeline类的对象
:param crawler:
:return:
"""
# crawler.settings 获取全部配置文件
path = crawler.settings.get('DB_PATH')
# 实例化对象
print('db from_crawl')
return cls(path) def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
print('db open_spider')
self.f = open(file=self.path, mode='a', encoding="utf-8") def process_item(self, item, spider):
"""
爬虫执行yield item对象,调用
:param item: item对象
:param spider: 爬虫对象
:return:
"""
print('db process_item value is {}'.format(item))
# self.f.write(item['text'] + '\n')
return item def close_spider(self, spider):
"""
爬虫结束,调用
:param spider:
:return:
"""
print('db close_spider')
self.f.close()

没有返回item

2)、DropItem

不执行后续pipeline类中的process_item方法

导入

from scrapy.exceptions import DropItem
# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
from scrapy.exceptions import DropItem class ToscrapyPipeline(object): def __init__(self, path):
self.f = None
self.path = path @classmethod
def from_crawler(cls, crawler):
"""
初始化方法时,创建pipeline类的对象
:param crawler:
:return:
"""
# crawler.settings 获取全部配置文件
path = crawler.settings.get('FILE_PATH')
# 实例化对象
print('file from_crawl')
return cls(path) def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
print('file open_spider')
self.f = open(file=self.path, mode='a', encoding="utf-8") def process_item(self, item, spider):
"""
爬虫执行yield item对象,调用
:param item: item对象
:param spider: 爬虫对象
:return:
"""
print('file process_item')
# self.f.write(item['text'] + '\n')
# return item
raise DropItem() def close_spider(self, spider):
"""
爬虫结束,调用
:param spider:
:return:
"""
print('file close_spider')
self.f.close() class DBPipeline(object): def __init__(self, path):
self.f = None
self.path = path @classmethod
def from_crawler(cls, crawler):
"""
初始化方法时,创建pipeline类的对象
:param crawler:
:return:
"""
# crawler.settings 获取全部配置文件
path = crawler.settings.get('DB_PATH')
# 实例化对象
print('db from_crawl')
return cls(path) def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
print('db open_spider')
self.f = open(file=self.path, mode='a', encoding="utf-8") def process_item(self, item, spider):
"""
爬虫执行yield item对象,调用
:param item: item对象
:param spider: 爬虫对象
:return:
"""
print('db process_item value is {}'.format(item))
# self.f.write(item['text'] + '\n')
return item def close_spider(self, spider):
"""
爬虫结束,调用
:param spider:
:return:
"""
print('db close_spider')
self.f.close()

DropItem

5、spider参数的作用

作用:pipelines.py中的类和方法是所有爬虫共用的

应用场景:如果先让某个方法,只有一个爬虫可以使用,就要用到spider参数

注意:spider参数对应的是爬虫中的name值

    def open_spider(self, spider):
"""
开始爬虫,调用
:param spider:
:return:
"""
#
if spider == 'quote':
print('file open_spider')
self.f = open(file=self.path, mode='a', encoding="utf-8")

持久化到redis可参考

https://www.cnblogs.com/wanglan/p/10826678.html

使用mongodb参考

https://blog.csdn.net/qq_41020281/article/details/79459604

05-22 04:03
查看更多