用生产者消费模型爬取智联招聘python岗位信息

爬取python岗位智联招聘

• 这里爬取北京地区岗位招聘python岗位，并存入EXECEL文件内，代码如下：

import json

import xlwt

import requests

from queue import Queue

from threading import Thread

def producer(q,path):

    res1 = requests.get(path,

                        headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.25 Safari/537.36 Core/1.70.3676.400 QQBrowser/10.4.3469.400'},

                        )

    result = json.loads(res1.text)

    for item in result["data"]["results"]:

        jobName = item["jobName"]

        companyname = item["company"]["name"]

        type = item["company"]["type"]["name"]

        size = item["company"]["size"]["name"]

        url = item["company"]['url']

        city = item['city']["items"][0]["name"]

        updateDate = item["updateDate"]

        salary = item["salary"]

        eduLevel = item["eduLevel"]["name"]

        workingExp = item["workingExp"]["name"]

        emplType = item["emplType"]

        condition = item["timeState"]

        msg = "%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s"%(jobName,companyname,type,size,url,city,updateDate,salary,eduLevel,workingExp,emplType,condition)

        q.put(msg)

def consumer(q):

    count = 1

    workbook = xlwt.Workbook(encoding="utf-8")

    worksheet = workbook.add_sheet(r"智联招聘")

    pattern = xlwt.Pattern()

    pattern.pattern = xlwt.Pattern.SOLID_PATTERN

    pattern.pattern_fore_colour = 2

    style = xlwt.XFStyle()

    style.pattern = pattern

    worksheet.write(0,0,r"工作名称", style)

    worksheet.write(0,1,r"公司",style)

    worksheet.write(0,2,r"企业性质",style)

    worksheet.write(0,3,r"人数",style)

    worksheet.write(0,4,r"链接网站",style)

    worksheet.write(0,5,r"所在地",style)

    worksheet.write(0,6,r"发布日期",style)

    worksheet.write(0,7,r"薪资",style)

    worksheet.write(0,8,r"学历",style)

    worksheet.write(0,9,r"工作年限",style)

    worksheet.write(0,10,r"职位",style)

    worksheet.write(0,11,r"状态",style)

    while True:

        msg = q.get()

        if not msg:

            workbook.save("智联招聘python信息.xls")

            break

        msg_list = msg.split("|")

        for index in range(len(msg_list)):

            worksheet.write(count, index, label=msg_list[index])

        count += 1

def cp(c_count,p_count):

    q = Queue(10)

    for i in range(c_count):

        Thread(target=consumer, args=(q,)).start()

    p_l = []

    for i in range(p_count,):

        path = "https://fe-api.zhaopin.com/c/i/sou?start=%d&pageSize=90&cityId=530&salary=0,0&workExperience=-1&education=-1&companyType=-1&employmentType=-1&jobWelfareTag=-1&kw=python&kt=3&=0&_v=0.55208065&x-zp-page-request-id=53d546acdd2a4e369735f9cb3a372e05-1560873187327-646177&x-zp-client-id=dc9fd9ff-68eb-42ab-8bc3-796640e734b3" % (

                i * 90)

        p1 = Thread(target=producer, args=(q,path))

        p1.start()

        p_l.append(p1)

    for p in p_l: p.join()

    for i in range(c_count):q.put(None)

if __name__ == '__main__':

    cp(1,12)

• 注意：此文章只用于学术交流