网易新网 spider

# -*- coding: utf-8 -*-

import os

import sys

import urllib.request

import requests

import re

from lxml import etree

def StringListSave(save_path, filename, slist):

    if not os.path.exists(save_path):

        os.makedirs(save_path)

    path = save_path+"/"+filename+".txt"

    with open(path, "w+") as fp:

        for s in slist:

            fp.write("%s\t\t%s\n" % (s[0].encode("utf8"), s[1].encode("utf8")))

def Page_Info(myPage):

    '''Regex'''

    mypage_Info = re.findall(r'<div class="titleBar" id=".*?"><h2>(.*?)</h2><div class="more"><a href="(.*?)">.*?</a></div></div>', myPage, re.S)

    return mypage_Info

def New_Page_Info(new_page):

    '''Regex(slowly) or Xpath(fast)'''

    # new_page_Info = re.findall(r'<td class=".*?">.*?<a href="(.*?)\.html".*?>(.*?)</a></td>', new_page, re.S)

    # # new_page_Info = re.findall(r'<td class=".*?">.*?<a href="(.*?)">(.*?)</a></td>', new_page, re.S) # bugs

    # results = []

    # for url, item in new_page_Info:

    #     results.append((item, url+".html"))

    # return results

    dom = etree.HTML(new_page)

    new_items = dom.xpath('//tr/td/a/text()')

    new_urls = dom.xpath('//tr/td/a/@href')

    assert(len(new_items) == len(new_urls))

    return zip(new_items, new_urls)

def Spider(url):

    i = 0

    print ("downloading ", url)

    myPage = requests.get(url).content.decode("gbk")

    # myPage = urllib.request.urlopen(url).read().decode("gbk")

    myPageResults = Page_Info(myPage)

    save_path = u"网易新闻抓取"

    filename = str(i)+"_"+u"新闻排行榜"

    StringListSave(save_path, filename, myPageResults)

    i += 1

    for item, url in myPageResults:

        print ("downloading ", url)

        new_page = requests.get(url).content.decode("gbk")

        # new_page = urllib.request.urlopen(url).read().decode("gbk")

        newPageResults = New_Page_Info(new_page)

        filename = str(i)+"_"+item

        StringListSave(save_path, filename, newPageResults)

        i += 1

if __name__ == '__main__':

    print ("start")

    start_url = "http://news.163.com/rank/"

    Spider(start_url)

    print ("end")