python 爬虫系列03--职位爬虫

职位爬虫

import requests

from lxml import etree

cookie = {

    'Cookie':'user_trace_token=20181015184304-692c4bf4-4e71-4cfd-8906-6219253e0ae8; _ga=GA1.2.1135099826.1539600208; LGUID=20181015184305-18c8e815-d067-11e8-bc15-5254005c3644; index_location_city=%E5%85%A8%E5%9B%BD; _gid=GA1.2.73712408.1539738633; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221667fc4129f205-01a02c2a87905b-51422e1f-2073600-1667fc412a0a16%22%2C%22%24device_id%22%3A%221667fc4129f205-01a02c2a87905b-51422e1f-2073600-1667fc412a0a16%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; JSESSIONID=ABAAABAAADEAAFI1F6DEB9C84C5A5AADBE0CCBE43481EB7; _gat=1; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1539600208,1539738633,1539769054; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1539769054; LGSID=20181017173710-3879f572-d1f0-11e8-bb7a-525400f775ce; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Fjobs%2Flist_%25E8%25BF%2590%25E7%25BB%25B4%25E5%25B7%25A5%25E7%25A8%258B%25E5%25B8%2588%3Fcity%3D%25E4%25B8%258A%25E6%25B5%25B7%26cl%3Dfalse%26fromSearch%3Dtrue%26labelWords%3D%26suginput%3D; LGRID=20181017173710-3879f6d3-d1f0-11e8-bb7a-525400f775ce; SEARCH_ID=47902a4acdc34c47977e8eeb46c523f2'

}

headers = {

    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36',

    'Accept':'application/json, text/javascript, */*; q=0.01',

    'Host':'www.lagou.com',

    'Origin':'https://www.lagou.com',

    'Referer':'https://www.lagou.com/jobs/list_%E8%BF%90%E7%BB%B4%E5%B7%A5%E7%A8%8B%E5%B8%88?city=%E4%B8%8A%E6%B5%B7&cl=false&fromSearch=true&labelWords=&suginput=',

}

data = {

    'first': False,

    'pn': 1,

    'kd': '运维工程师',

}

def get_job(data):

   # url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E6%B7%B1%E5%9C%B3&needAddtionalResult=false&isSchoolJob=0'

    url = 'https://www.lagou.com/jobs/positionAjax.json?city=%E4%B8%8A%E6%B5%B7&needAddtionalResult=false'

    page = requests.post(url=url, cookies=cookie, headers=headers, data=data)

    page.encoding = 'utf-8'

    result = page.json()

    jobs = result['content']['positionResult']['result']

    for job in jobs:

        companyShortName = job['companyShortName']

        positionId = job['positionId']  # 主页ID

        companyFullName = job['companyFullName'] # 公司全名

        companyLabelList = job['companyLabelList'] # 福利待遇

        companySize = job['companySize'] # 公司规模

        industryField = job['industryField']

        createTime = job['createTime'] # 发布时间

        district = job['district'] # 地区

        education = job['education'] # 学历要求

        financeStage = job['financeStage'] # 上市否

        firstType = job['firstType'] # 类型

        secondType = job['secondType'] # 类型

        formatCreateTime = job['formatCreateTime']

        publisherId = job['publisherId'] # 发布人ID

        salary = job['salary'] # 薪资

        workYear = job['workYear'] # 工作年限

        positionName = job['positionName'] #

        jobNature = job['jobNature'] # 全职

        positionAdvantage = job['positionAdvantage'] # 工作福利

        positionLables = job['positionLables'] # 工种

        detail_url = 'https://www.lagou.com/jobs/{}.html'.format(positionId)

        response = requests.get(url=detail_url, headers=headers, cookies=cookie)

        response.encoding = 'utf-8'

        tree = etree.HTML(response.text)

        desc = tree.xpath('//*[@id="job_detail"]/dd[2]/div/p/text()')

        print(companyFullName)

        print('%s 拉勾网链接:-> %s' % (companyShortName, detail_url))

        print('职位：%s' % positionName)

        print('职位类型：%s' % firstType)

        print('薪资待遇：%s' % salary)

        print('职位诱惑：%s' % positionAdvantage)

        print('地区：%s' % district)

        print('类型：%s' % jobNature)

        print('工作经验：%s' % workYear)

        print('学历要求：%s' % education)

        print('发布时间：%s' % createTime)

        x = ''

        for label in positionLables:

            x += label + ','

        print('技能标签：%s' % x)

        print('公司类型：%s' % industryField)

        for des in desc:

            print(des)

def url(data):

    for x in range(1,50):

        data['pn'] = x

        get_job(data)

if __name__ == '__main__':

    url(data)