Python 2.7 爬取51job 全国java岗位

 

一页有50条数据一共2000页 分页是get分页

#!/usr/bin/python

# encoding: utf-8

import requests

import threading

from lxml import etree

import sys

import os

import datetime

import re

import random

import time

reload(sys)

sys.setdefaultencoding('utf-8')

# 定义写入日志的方法

def log(context):

    txtName = "./log/log.txt"

    f=file(txtName, "a+")

    f.writelines(context+"\n")

    f.close()

def xin():

        # 请求头

        header = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',

            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

            'Accept-Encoding': 'gzip, deflate, br',

            'Accept-Language': 'zh-CN,zh;q=0.9'

        }

        count=1

           # 一共2000页

        while (count < 2000):

            url="https://search.51job.com/list/000000,000000,0000,00,9,99,java,2,"+str(count)+".html?lang=c&stype=1&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare="

            response=requests.get(url,headers=header)

            html=response.content.decode("gbk")

            print(html)

            selector=etree.HTML(html)

            contents = selector.xpath('//div[@class="dw_table"]/div[@class="el"]')

            log("第"+str(count)+"页了--"+str(len(contents))+"条数据")

            for eachlink in contents:

                    company = eachlink.xpath('span[@class="t2"]/a/text()')[0]

                    url= eachlink.xpath('p/span/a/@href')[0]

                    name= eachlink.xpath('p/span/a/text()')[0]

                    city= eachlink.xpath('span[@class="t3"]/text()')[0]

                    # 工资有的是没有的

                    key=""

                    if len(eachlink.xpath('span[@class="t4"]/text()'))<1:

                        key=""

                    else:

                        key= eachlink.xpath('span[@class="t4"]/text()')[0]

                    # 把空格去掉

                    company=company.replace(' ','')

                    name=name.replace(' ','')

                    city=city.replace(' ','')

                    zhi=name+"============="+company+"============="+city+"============="+str(key)+"============="+url

                    txtName = "./file/java.txt"

                    f=file(txtName, "a+")

                    f.write(zhi)

                    f.close()

            sui=random.randint(1,5)

            log("休眠"+str(sui))

            time.sleep(sui)

            count=count+1

if __name__=="__main__":

    xin()

日志文件

爬去的数据

但是爬去的速度有点慢,

于是乎采用了多线程爬去,

但是51job 立刻就把IP段给封掉了,

于是用户4台服务器,每台爬取500条数据,最后再结合一起加到数据库中

人生苦短,我用Python!!!