20.Scrapy日常练手

1.创建爬虫项目：

scrapy  startproject tutorial

2.创建 spider

cd tutorial

scrapy  genspider quotes quotes.toscrape.com

如下图：

3.

quotes.py

___________________________________________________________________________

 # -*- coding: utf-8 -*-

 import scrapy

 from tutorial.items import TutorialItem

 import logging

 class QuotesSpider(scrapy.Spider):

     name = 'quotes'

     allowed_domains = ['quotes.toscrape.com']

     start_urls = ['http://quotes.toscrape.com/']

     def parse(self, response):

         quotes=response.css('.quote')

         for quote in quotes:

             item=TutorialItem()

             #内容

             item['text']=quote.css('.text::text').extract_first()

             #作者

             item['author']=quote.css('.author::text').extract_first()

             #标签

             item['tags']=quote.css('.tags .tag::text').extract_first()

             yield item

         #下一页

         next=response.css('.pager .next a::attr("href")').extract_first()

         url=response.urljoin(next)

         yield scrapy.Request(url=url,callback=self.parse)

items.py
________________________________________________________________________

 # -*- coding: utf-8 -*-

 # Define here the models for your scraped items

 #

 # See documentation in:

 # https://doc.scrapy.org/en/latest/topics/items.html

 import scrapy

 class TutorialItem(scrapy.Item):

     # define the fields for your item here like:

     # name = scrapy.Field()

     text=scrapy.Field()

     author=scrapy.Field()

     tags=scrapy.Field()

 piplines.py

_________________________________________________________________________

 # -*- coding: utf-8 -*-

 # Define your item pipelines here

 #

 # Don't forget to add your pipeline to the ITEM_PIPELINES setting

 # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html

 from scrapy.exceptions import DropItem

 import pymysql

 class TutorialPipeline(object):

     # def __init__(self):

     #     self.limit=50

     # def process_item(self, item, spider):

     #     if  item['text']:

     #         if len(item['text'])>self.limit:

     #             item['text']=item['text'][0:self.limit].rstrip()+'...'

     #         return item

     #     else:

     #         return DropItem('Missing Text')

     def __init__(self):

         pass

     def open_spider(self, spider):

         self.my_conn = pymysql.connect(

             host = '192.168.113.129',

             port = 3306,

             database = 'datas',

             user = 'root',

             password = '',

             charset = 'utf8'

         )

         self.my_cursor = self.my_conn.cursor()

     def process_item(self,item, spider):

         dict(item)

         insert_sql = "insert into quotes(author,tags,text) values(%s,%s,%s)"

         self.my_cursor.execute(insert_sql,[item['author'],item['tags'],item['text']])

         return  item

     def close_spider(self, spider):

         self.my_conn.commit()

         self.my_cursor.close()

         self.my_conn.close()

setting.py
___________________________________________________________________________

# Obey robots.txt rules

ROBOTSTXT_OBEY = True

ITEM_PIPELINES = {

   'tutorial.pipelines.TutorialPipeline': 200,

}

代码配置完：

保存文件格式

scrapy crawl  quotes -o quotes.xml

scrapy crawl  quotes -o quotes.csv