拉钩网爬取所有python职位信息

最近在找工作，所以爬取了拉钩网的全部python职位，以便给自己提供一个方向。拉钩网的数据还是比较容易爬取的，得到json数据直接解析就行，废话不多说， 直接贴代码：

 import json

 import urllib

 import urllib2

 from openpyxl import load_workbook

 filename = 'E:\excel\position_number_11_2.xlsx'

 ws = load_workbook(filename=filename)

 sheet = ws.create_sheet(0)

 sheet.title = 'position'

 count = 1

 for page in xrange(100):

     from_data = {

         'first': 'false',

         'pn': page,

         'kd': 'Python'

     }

     header = {

         "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0',

         'Referer': 'https://www.lagou.com/jobs/list_Python?px=default&city=%E5%85%A8%E5%9B%BD',

     }

     request_url = 'https://www.lagou.com/jobs/positionAjax.json?px=default&needAddtionalResult=false'

     data = urllib.urlencode(from_data)

     request = urllib2.Request(request_url, headers=header, data=data)

         try:

         html = urllib2.urlopen(request).read().decode('utf-8')

     except Exception:

         print '没有职位信息'

         break

     # print html

     jsonobj = json.loads(html)

     # print jsonobj

     dict_obj = jsonobj['content']['positionResult']['result']

     for item in dict_obj:

         if item:

             sheet.cell(row=count, column=1).value = item['companySize']

             sheet.cell(row=count, column=2).value = item['workYear']

             sheet.cell(row=count, column=3).value = item['education']

             sheet.cell(row=count, column=4).value = item['financeStage']

             sheet.cell(row=count, column=5).value = item['city']

             sheet.cell(row=count, column=6).value = item['industryField']

             sheet.cell(row=count, column=7).value = item['formatCreateTime']

             sheet.cell(row=count, column=8).value = item['positionName']

             sheet.cell(row=count, column=9).value = item['companyFullName']

             sheet.cell(row=count, column=10).value = item['salary']

             count += 1

             ws.save('E:\excel\position_number_11_2.xlsx')

代码写得比较急，就没怎么规范。 过两天把微博和豆瓣的代码发出来，希望园里的大神多指点^_^