python2 urllib2抓取51job网的招聘数据

 #coding=utf-8

 __author__ = "carry"

 import sys

 reload(sys)

 sys.setdefaultencoding('utf-8')

 import urllib

 import urllib2

 import re

 #获取源码

 def get_content(page):

     headers = {#'Host':'search.51job.com',

                'User-Agent':'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0',

                #'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',

                #'Connection':'keep-alive'

                }

     url ='http://search.51job.com/list/000000,000000,0000,00,9,99,python,2,'+ str(page)+'.html'

     req = urllib2.Request(url,headers=headers)

     r = urllib2.urlopen(req)

     response = r.read() #读取源代码并转为unicode

     html = response.decode('gbk').encode('utf-8')

     return html

 def get(html):

     reg = re.compile(r'class="t1 ">.*? <a target="_blank" title="(.*?)".*? <span class="t2"><a target="_blank" title="(.*?)".*?<span class="t3">(.*?)</span>.*?<span class="t4">(.*?)</span>.*? <span class="t5">(.*?)</span>',re.S)#匹配换行符

     items=re.findall(reg,html)

     return items

 #多页处理，下载到文件

 for  j in range(1,11):

     print(u"正在爬取第"+str(j)+"页数据...")

     html = get_content(j) #调用获取网页原码

     for i in get(html):

         #print(i[0],i[1],i[2],i[3],i[4])

         with open ('51job.txt','a') as f:

             f.write(i[0]+'\t'+i[1]+'\t'+i[2]+'\t'+i[3]+'\t'+i[4]+'\n')

             f.write("-----------------------------------------------------")

             f.close()