<爬虫>利用BeautifulSoup爬取百度百科虚拟人物资料存入Mysql数据库

网页情况:

代码:

import requests

from requests.exceptions import RequestException

from bs4 import BeautifulSoup as bs

import re

import time

import pymysql

def get_one_page(url):

    #得到一页的内容

    try:

        headers  = {

            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'

        }

        response = requests.get(url,headers=headers)

        response.encoding = 'utf-8'

        if response.status_code == 200:

            return response.text

        return None

    except RequestException:

        return None

def parse_one_page(html):

    #使用BeautifuSoup解析一页的内容

    soup = bs(html,'lxml')

    for data in soup.find_all('div',class_="photo"):

        name = data.a['title']

        href = "https://baike.baidu.com"+data.a['href']

        img = data.img['src']

        #有些人物没有图片,图片链接有误需要拼接

        if re.search("^/static",img):

            img ="https://baike.baidu.com" + img

        yield {

            'name':name,

            'href':href,

            'img':img

        }

def write_mysql(item):

    #写入Mysql数据库

    conn = pymysql.connect(

        host='localhost',

        user='root',

        password='',

        database='baidu',

        charset='utf8'  # 别写成utf-8

    )

    cursor = conn.cursor()  # 建立游标

    sql = "insert into baidu_baike(name,href,img) values(%s,%s,%s)"

    cursor.execute(sql,(item['name'],item['href'],item['img']))  # 注意excute的位置参数的问题

    conn.commit()  # 修改值的时候,一定需要commit

    cursor.close()  # 关闭

    conn.close()  # 关闭

def main(url):

    #主函数

    html = get_one_page(url)

    items = parse_one_page(html)

    for item in items:

        write_mysql(item)

if __name__ == '__main__':

    #分析URL构成,拼接URL

    for i in range(1,7):

        url = "http://baike.baidu.com/fenlei/虚拟人物?limit=30&index=" + str(i) + "&offset=" + str(

            30 * (int(i) - 1)) + "# gotoList"

        main(url)

        print('正在爬取第%s页'%i)

        time.sleep(1)

    print("全部写入成功!")

　　运行结果:

mysql数据库结果: