Python的scrapy之爬取51job网站的职位

今天老师讲解了Python中的爬虫框架--scrapy，然后带领我们做了一个小爬虫--爬取51job网的职位信息，并且保存到数据库中

用的是Python3.6  pycharm编辑器

爬虫主体：

import scrapy

from ..items import JobspidersItem

class JobsspiderSpider(scrapy.Spider):

    name = 'jobsspider'

    #allowed_domains = ['search.51job.com/list/010000,000000,0000,00,9,99,%2520,2,1.html']

    #start_urls = ['https://search.51job.com/list/010000,000000,0000,00,9,99,%2520,2,1.html/']

    start_urls = [

        'https://search.51job.com/list/010000,000000,0000,01,9,99,python,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=']

    def parse(self, response):

        currentPageItems = response.xpath('/html/body/div[@class="dw_wp"]/div[@class="dw_table"]/div[@class="el"]')

        print(currentPageItems)

        # currentPageItems = response.xpath('//div[@class="el"]')

        for jobItem in currentPageItems:

            print('----',jobItem)

            jobspidersItem = JobspidersItem()

            jobPosition = jobItem.xpath('p[@class="t1 "]/span/a/text()').extract()

            if jobPosition:

                #print(jobPosition[0].strip())

                jobspidersItem['jobPosition'] = jobPosition[0].strip()

            jobCompany = jobItem.xpath('span[@class="t2"]/a/text()').extract()

            if jobCompany:

                #print(jobCompany[0].strip())

                jobspidersItem['jobCompany'] = jobCompany[0].strip()

            jobArea = jobItem.xpath('span[@class="t3"]/text()').extract()

            if jobArea:

                #print(jobArea[0].strip())

                jobspidersItem['jobArea'] = jobArea[0].strip()

            jobSale = jobItem.xpath('span[@class="t4"]/text()').extract()

            if jobSale:

                # print(jobCompany[0].strip())

                jobspidersItem['jobSale'] = jobSale[0].strip()

            jobDate = jobItem.xpath('span[@class="t5"]/text()').extract()

            if jobDate:

                # print(jobCompany[0].strip())

                jobspidersItem['jobDate'] = jobDate[0].strip()

            yield jobspidersItem  # 通过yield 调用输出管道

            pass

        nextPageURL = response.xpath('//li[@class="bk"]/a/@href').extract()  # 取下一页的地址

        print(nextPageURL)

        if nextPageURL:

            url = response.urljoin(nextPageURL[-1])

            print('url', url)

            # 发送下一页请求并调用parse()函数继续解析

            yield scrapy.Request(url, self.parse, dont_filter=False)

            pass

        else:

            print("退出")

        pass

items.py   设置五个items

import scrapy

class JobspidersItem(scrapy.Item):

    # define the fields for your item here like:

    jobPosition = scrapy.Field()

    jobCompany = scrapy.Field()

    jobArea = scrapy.Field()

    jobSale = scrapy.Field()

    jobDate = scrapy.Field()

    pass

pipelines.py  输出管道

class JobspidersPipeline(object):

    def process_item(self, item, spider):

        print('职位:', item['jobPosition'])

        print('公司:', item['jobCompany'])

        print('工作地点:', item['jobArea'])

        print('薪资:', item['jobSale'])

        print('发布时间:', item['jobDate'])

        print('----------------------------')

        return item

pipelinesmysql.py  输出到mysql中  第一行的意思是使用了以前封装的数据库操作类

from week5_day04.dbutil import dbutil

# 作业： 自定义的管道，将完整的爬取数据，保存到MySql数据库中

class JobspidersPipeline(object):

    def process_item(self, item, spider):

        dbu = dbutil.MYSQLdbUtil()

        dbu.getConnection()  # 开启事物

        # 1.添加

        try:

            #sql = "insert into jobs (职位名,公司名,工作地点,薪资,发布时间)values(%s,%s,%s,%s,%s)"

            sql = "insert into t_job (jobname,jobcompany,jobarea,jobsale,jobdata)values(%s,%s,%s,%s,%s)"

            #date = []

            #dbu.execute(sql, date, True)

            dbu.execute(sql, (item['jobPosition'],item['jobCompany'],item['jobArea'],item['jobSale'],item['jobDate']),True)

            #dbu.execute(sql,True)

            dbu.commit()

            print('插入数据库成功！！')

        except:

            dbu.rollback()

            dbu.commit()  # 回滚后要提交

        finally:

            dbu.close()

        return item

最终结果：

通过这个最基础的51job爬虫，进入到scrapy框架的学习中，这东西挺好使