scrapy请求传参-BOSS反爬

scrapy请求传参-BOSS反爬

思路总结

首先boss加了反爬

是cookies的

爬取的内容为职位和职位描述

# -*- coding: utf-8 -*-
import scrapy
from boss.items import BossItem


class Boss01Spider(scrapy.Spider):
    name = 'boss_01'
    start_urls = ['https://www.zhipin.com/c101210100/?query=python&page=1']

    #修改第一次请求
    def start_requests(self):
        cookies = "自己填写自己的cookies"
        cookies = {i.split("=")[0]: i.split("=")[1] for i in cookies.split("; ")}
        yield scrapy.Request(
            self.start_urls[0],
            callback=self.parse,
            cookies=cookies
        )

    def parse_detail(self,response):
        data=response.xpath('//*[@id="main"]/div[3]/div/div[2]/div[2]/div[1]/div/text()').extract()
        all_data=''
        for i in data:
            all_data+="\n"+i
        item = response.meta['item']
        item['job_content']=all_data
        yield item

    def parse(self, response):
        job_list=response.xpath('//div[@class="job-list"]/ul/li')
        for i in job_list:
            item = BossItem()
            #爬取岗位名称和详情的url
            job_name=i.xpath('./div/div/h3/a/div[@class="job-title"]/text()').extract_first()
            job_url=i.xpath('./div/div/h3/a/@href').extract_first()
            job_all_url="https://www.zhipin.com"+job_url
            item['job_name']=job_name
            yield scrapy.Request(job_all_url, callback=self.parse_detail,meta={'item':item})