pipeline的四个方法
@classmethod
def from_crawler(cls, crawler):
"""
初始化的时候，用以创建pipeline对象
:param crawler:
:return:
"""
pass def open_spider(self, spider):
"""
爬虫开始执行时，调用
:param spider:
:return:
"""
pass def process_item(self, item, spider):
"""
每当数据需要持久化时，就会被调用
:param item:
:param spider:
:return:
""" return item def close_spider(self, spider):
"""
爬虫结束执行时，调用
:param spider:
:return:
"""
pass
实例
import pymysql
from scrapy.exceptions import DropItem class ChoutiPipeline(object):
def __init__(self, db_conf):
self.db_conf = db_conf
self.conn = None
self.cursor = None @classmethod
def from_crawler(cls, crawler):
"""
初始化的时候，用以创建pipeline对象
:param crawler:
:return:
"""
db_conf = crawler.settings.get('DATABASE')
return cls(db_conf) def open_spider(self, spider):
"""
爬虫开始执行时，调用
:param spider:
:return:
"""
print('爬虫开始 ...')
self.conn = pymysql.connect(
host=self.db_conf['host'],
port=self.db_conf['port'],
user=self.db_conf['user'],
passwd=self.db_conf['password'],
db=self.db_conf['db'],
charset=self.db_conf['charset']
)
self.cursor = self.conn.cursor() def process_item(self, item, spider):
"""
每当数据需要持久化时，就会被调用
:param item:
:param spider:
:return:
""" sql = 'INSERT INTO articles(title, title_url, summary, create_time, url_md5)' \
' VALUES ("%s", "%s" ,"%s", "%s", "%s")' a = sql % (item['title'], item['title_url'], item['summary'], item['create_time'], item['url_md5']) try:
self.cursor.execute(a)
self.conn.commit()
except Exception as e:
print(e)
return DropItem() def close_spider(self, spider):
"""
爬虫结束执行时，调用
:param spider:
:return:
"""
self.cursor.close()
self.conn.close()
print('爬虫结束 ...')
注册配置文件
全局配置：
# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'day1.pipelines.ChoutiPipeline': 300,
}
也可以控制某个爬虫执行那行那个pipeline
class ChoutiSpider(scrapy.Spider):
name = 'Chouti'
allowed_domains = ['dig.chouti.com']
start_urls = ['https://dig.chouti.com/'] custom_settings = {
'ITEM_PIPELINES': {'day1.pipelines.ChoutiPipeline': 1}
}