爬虫之爬取斗鱼官网LOL部分主播的状态

一个爬虫小程序 爬取主播的排名及观看人数

import re

import requests

import request

class Spider():

    url = 'https://www.douyu.com/g_lol'

    root_pattern = '<p>([\s\S]*?)</p>'

    name_pattern = '<span class="dy-name ellipsis fl">([\s\S]*?)</span>'

    number_pattern = '<span class="dy-num fr"  >([\s\S]*?)</span>'

    def __fetch_content(self):

        r = requests.get(Spider.url)

        htmls = r.text

        return htmls

    def __analysis(self, htmls):

        root_htmls = re.findall(Spider.root_pattern, htmls)

        anchors = []

        for html in root_htmls:

            name = re.findall(Spider.name_pattern, html)

            number = re.findall(Spider.number_pattern, html)

            anchor = {'name': name, 'number': number}

            anchors.append(anchor)

        return anchors

    def __refine(self, anchors):

        l = lambda anchor: {

            'name': anchor['name'][0],

            'number': anchor['number'][0]

            }

        return map(l, anchors)

    def __sort(self, anchors):

        anchors = sorted(anchors, key=self.__sort_seed, reverse=True)

        return anchors

    def __sort_seed(self, anchor):

        r = re.findall('\d*', anchor['number'])

        number = float(r[0])

        if '万' in anchor['number']:

            number *= 10000

        return number

    def __show(self, anchors):

        for rank in range(0, len(anchors)):

            print(

                '人数排名' + str(rank + 1)

                + ' : ' + anchors[rank]['name']

                + '~~~~~~' + anchors[rank]['number']

            )

    def go(self):

        htmls = self.__fetch_content()

        anchors = self.__analysis(htmls)

        anchors = list(self.__refine(anchors))

        anchors = self.__sort(anchors)

        self.__show(anchors)

spider = Spider()

spider.go()

运行结果：

喜欢的朋友们可以去看主播的排名啦