Python爬虫，爬取实验楼全部课程

目的：

使用requests库以及xpath解析进行实验楼所有课程，存入MySQL数据 库中。

准备工作：

首先安装，requests库，lxml库，以及peewee库。在命令行模式，使用以下命令。

pip install requests

pip install lxml

pip install peewee

　　然后，就可以打开编辑器编写代码了。

代码:

 # 引入所需要的库

 import time

 import requests

 from peewee import *

 from lxml import etree

 # 这个程序要倒着看

 # 这个是连接数据库的，名字和密码根据自己情况修改

 db = MySQLDatabase('shiyanlou', user='root', passwd='xxxxxx')

 class Course(Model):

     title = CharField()

     teacher = CharField()

     teacher_courses = IntegerField()

     tag = CharField()

     study_num = IntegerField()

     content = CharField()

     class Meta:

         database = db

 Course.create_table()

 def parse_content(url, title, tag, study_num):

     print('课程地址：' + url)

     res = requests.get(url)

     xml = etree.HTML(res.text)

     # 获取页面里的简介

     try:

         content = xml.xpath('//meta[@name="description"]/@content')[0]

     except Exception as e:

         content = '无'

     # 获取老师名字

     try:

         teacher = xml.xpath(

             '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="name"]/strong/text()')[0]

     except Exception as e:

         teacher = '匿名'

     # 获取老师发表课程数目

     try:

         teacher_courses = xml.xpath(

             '//div[@class="sidebox mooc-teacher"]//div[@class="mooc-info"]/div[@class="courses"]/strong/text()')[0]

     except Exception as e:

         teacher_courses = '未知'

     # 存入数据库

     try:

         course = Course(title=title, teacher=teacher,

                         teacher_courses=int(teacher_courses), tag=tag, study_num=int(study_num), content=content)

         course.save()

     except Exception as e:

         print('一条数据存取失败')

 def get_course_link(url):

     # 获取每一页的信息，传给下一个函数

     response = requests.get(url)

     xml = etree.HTML(response.text)

     # contains()是包含的意思

     courses = xml.xpath(

         '//div[contains(@class, "col-md-3") and contains(@class, "col-sm-6") and contains(@class, "course")]')

     for course in courses:

         try:

             url = 'https://www.shiyanlou.com' + course.xpath('.//a/@href')[0]

         except Exception as e:

             print('一个课程页面未获得')

             continue

         title = course.xpath('.//div[@class="course-name"]/text()')[0]

         study_people = course.xpath(

             './/span[@class="course-per-num pull-left"]/text()')[1].strip()

         # study_people = int(study_people)

         try:

             tag = course.xpath(

                 './/span[@class="course-money pull-right"]/text()')[0]

         except Exception as e:

             tag = "普通"

         parse_content(url=url, title=title, tag=tag, study_num=study_people)

         # time.sleep(0.5)

 def main():

     # 通过requests库的get获得目标地址的返回信息，类型为Response

     response = requests.get('https://www.shiyanlou.com/courses/')

     # 将返回信息的文本转化为xml树，可以通过xpath来进行查询

     xml = etree.HTML(response.text)

     # 由分析网页源代码可以总结，url分页模式，只有最后的数字不一样

     course_link = 'https://www.shiyanlou.com/courses/?category=all&course_type=all&fee=all&tag=all&page={}'

     # 这里获得最大页数就可以了,xpath()函数里的便是寻找路径了

     # //会在全文来进行查找，//ul则是查找全文的ul标签，//ul[@class="pagination"]会仅查找有class属性，

     # 且为"pagination"的标签，之后/li是查找当前的ul标签下的li标签（仅取一层),取查询到的列表倒数第二个标签

     # 为li[last()-1],/a/text()查询a标签里的文本内容

     page = xml.xpath('//ul[@class="pagination"]/li[last()-1]/a/text()')

     if len(page) != 1:

         print('爬取最大页数时发生错误！！')

         return None

     # page原是一个列表，这里取出它的元素，并转化为Int型

     page = int(page[0])

     # 将每一页的url传给get_course_link函数进行处理

     for i in range(1, page + 1):

         # 填入course_link,获取完整url

         url = course_link.format(i)

         print('页面地址：' + url)

         # 调用另一个函数

         get_course_link(url)

 if __name__ == '__main__':

     # 调用main函数

     main()

 # [Finished in 218.5s]