从0开始学爬虫8使用requests/pymysql和beautifulsoup4爬取维基百科词条链接并存入数据库

从0开始学爬虫8使用requests和beautifulsoup4爬取维基百科词条链接并存入数据库

Python使用requests和beautifulsoup4爬取维基百科词条链接并存入数据库

参考文档：

https://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/

# 安装 beautifulsoup4

(pytools) D:\python\pytools>pip install beautifulsoup4

安装mysql的模块

pymysql的地址：https://github.com/PyMySQL/PyMySQL

爬取维基百科词条

# coding=utf-8

from bs4 import BeautifulSoup

import requests

import re

def spider_wike():

    url = "https://en.wikipedia.org/wiki/Main_Page"

    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}

    resp = requests.get(url, headers = headers)

    # 将响应数据转换为utf-8编码

    resp.encoding = 'utf-8'

    html_doc = resp.text

    soup = BeautifulSoup(html_doc, "html.parser")

    # 找到以wiki开头的a标签的href属性

    list_urls = soup.find_all("a", href=re.compile("^/wiki/"))

    # print(list_urls)

    # 输出所有的词条对应的名称和URL

    for url in list_urls:

        # 过滤掉.jpg 或.JPG 结尾的URL

        if not re.search(r"\.(jpg|JPG)", url["href"]):

            # 词条加网址

            # sting只能获取一个， get_text() 可以获取标签下所有的内容

            print(url.get_text(), " <------>", "https://en.wikipedia.org" + url["href"])

if __name__ == '__main__':

    spider_wike()

# 将维基百科词条链接存入数据库

# coding=utf-8

from bs4 import BeautifulSoup

import requests

import re

import pymysql.cursors

'''

    # 环境准备

    pip install pymysql

    create database wikiurl charset=utf8mb4;

    use wikiurl;

    create table urls (id int primary key auto_increment,urlname varchar(255),urlhref varchar(1000));

'''

url = "https://en.wikipedia.org/wiki/Main_Page"

headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36"}

resp = requests.get(url, headers = headers)

# 将响应数据转换为utf-8编码

resp.encoding = 'utf-8'

html_doc = resp.text

soup = BeautifulSoup(html_doc, "html.parser")

# 找到以wiki开头的a标签的href属性

list_urls = soup.find_all("a", href=re.compile("^/wiki/"))

# print(list_urls)

# 输出所有的词条对应的名称和URL

for url in list_urls:

    # 过滤掉.jpg 或.JPG 结尾的URL

    if not re.search(r"\.(jpg|JPG)", url["href"]):

        # 词条加网址

        # sting只能获取一个， get_text() 可以获取标签下所有的内容

        print(url.get_text(), " <------>", "https://en.wikipedia.org" + url["href"])

        connection = pymysql.connect(host='localhost',

                                     user='root',

                                     password='root',

                                     db='wikiurl',

                                     charset='utf8mb4')

        try:

            # 获取回话指针

            with connection.cursor() as cursor:

                # 创建sql语句

                sql = "insert into `urls`(`urlname`,`urlhref`) values(%s,%s)"

                # 执行sql语句

                cursor.execute(sql,(url.get_text(), "https://en.wikipedia.org" + url["href"]))

                # 提交数据

                connection.commit()

        finally:

            connection.close()

# 从数据库读取词条信息

# coding=utf-8

import pymysql

def get_conn():

    connection = pymysql.connect(host='localhost',

                                 user='root',

                                 password='root',

                                 db='wikiurl',

                                 charset='utf8mb4')

    return connection

def get_wiki_data():

    conn = get_conn()

    sql = "select `urlname`,`urlhref` from urls"

    cur = conn.cursor()

    # 获取总记录条数

    count = cur.execute(sql)

    print(count)

    # 获取所有数据

    # urllists = cur.fetchall()

    # 获取指定条目数据

    # urllists = cur.fetchmany(3)

    #

    # for url in urllists:

    #     print(url[0],'<--->',url[1])

    # 获取一条数据

    link = cur.fetchone()

    print(link)

    # 关闭数据库连接

    conn.close()

def get_data():

    conn = get_conn()

    try:

        with conn.cursor() as cur:

            sql = "select `urlname`,`urlhref` from urls where `id` is not NULL"

            count = cur.execute(sql)

            print(count)

            # 查询所有数据

            # data = cur.fetchall()

            # print(data)

            # 查询指定条目数据

            result = cur.fetchmany(size = 5)

            print(result)

    finally:

        conn.close()

if __name__ == '__main__':

    # get_wiki_data()

    get_data()