实现自动翻页功能

示例代码一

#!/usr/bin/env python
# -*- encoding: utf- -*-
# Created on -- ::
# Project: v2ex from pyspider.libs.base_handler import *
#import re class Handler(BaseHandler):
crawl_config = {
} @every(minutes= * )
def on_start(self):
self.crawl('https://www.v2ex.com/', callback=self.index_page, validate_cert=False) @config(age= * * * )
def index_page(self, response):
for each in response.doc('a[href^="https://www.v2ex.com/?tab="]').items():
self.crawl(each.attr.href, callback=self.tab_page, validate_cert=False) @config(age= * * * )
def tab_page(self, response):
for each in response.doc('a[href^="https://www.v2ex.com/go/"]').items():
self.crawl(each.attr.href, callback=self.board_page, validate_cert=False) @config(priority=)
def board_page(self, response):
#实现自动翻页功能
for each in response.doc('a[href^="https://www.v2ex.com/t/"]').items():
url = each.attr.href
if url.find('#reply')>:
url = url[:url.find('#')]
self.crawl(url, callback=self.detail_page, validate_cert=False)
for each in response.doc('a.page_normal').items():
self.crawl(each.attr.href, callback=self.board_page, validate_cert=False) @config(priority=)
def detail_page(self, response):
title = response.doc('h1').text()
content = response.doc('div.topic_content').html().replace('"', '\\"')
tmp = zip(response.doc('a[href^="/member/"]').items(), response.doc('div.reply_content').items())
reply_content = list()
for e1, e2 in tmp:
reply_content.append((e1.text(), e2.text()))
#self.add_question(title, content) #插入数据库
return {
"url": response.url,
"title": title,
"content": content,
"reply_content": reply_content,
}

示例代码二

#!/usr/bin/env python
# -*- encoding: utf- -*-
# Created on -- ::
# Project: tutorial_douban_movie import re
from pyspider.libs.base_handler import * class Handler(BaseHandler):
"""
This is a sample script for: pyspider 爬虫教程(一):HTML 和 CSS 选择器
http://blog.binux.me/2015/01/pyspider-tutorial-level-1-html-and-css-selector/
""" @every(minutes= * )
def on_start(self):
self.crawl('http://movie.douban.com/tag/', callback=self.index_page) @config(age= * * )
def index_page(self, response):
for each in response.doc('a[href^="http"]').items():
if 'tag' in each.attr.href:
self.crawl(each.attr.href, callback=self.list_page) @config(age=***, priority=)
def list_page(self, response):
for each in response.doc('HTML>BODY>DIV#wrapper>DIV#content>DIV.grid-16-8.clearfix>DIV.article>DIV>TABLE TR.item>TD>DIV.pl2>A').items():
self.crawl(each.attr.href, priority=, callback=self.detail_page)
# 翻页
for each in response.doc('HTML>BODY>DIV#wrapper>DIV#content>DIV.grid-16-8.clearfix>DIV.article>DIV.paginator>A').items():
self.crawl(each.attr.href, callback=self.list_page) @config(priority=)
def detail_page(self, response):
return {
"url": response.url,
"title": response.doc('HTML>BODY>DIV#wrapper>DIV#content>H1>SPAN').text(),
"rating": response.doc('#interest_sectl > div.rating_wrap.clearbox > div.rating_self.clearfix > strong').text(),
"导演": [x.text() for x in response.doc('a[rel="v:directedBy"]').items()],
}
05-06 11:42