requests爬取知乎话题和子话题

zhihu.py

# *_*coding:utf-8 *_*

import pymysql

import requests

from lxml import etree

from requests_test.child_topic import GetChildTopic

from requests_test.parent_topic import GetParentTopic

if __name__ == "__main__":

    parent = GetParentTopic()

    res = parent.get_parent_data()

    # child  = GetChildTopic()

    # child.get_child_data(1027,2)

    child = GetChildTopic()

    for i in res:

        print("parent_id:",i)

        child.get_child_data(i,50)

　　

parent_topic.py

# *_*coding:utf-8 *_*

import pymysql

from lxml import etree

import requests

class GetParentTopic(object):

    def __init__(self):

        self.conn = pymysql.connect(host='192.168.33.10', user='root', passwd='root', db='spider', charset='utf8')

        self.cur = self.conn.cursor()

    def get_parent_data(self):

        headers = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

        }

        url = 'https://www.zhihu.com/topics'

        response = requests.get(url, headers=headers)

        res = response.text

        html = etree.HTML(res)

        ul = html.xpath("//ul[@class='zm-topic-cat-main clearfix']/li");

        parent_topic = {}

        for li in ul:

            title = li.xpath('./a/text()')[0];

            topic_id = li.xpath('./@data-id')[0];

            parent_topic[topic_id] = title

            import time

            # 格式化成2016-03-20 11:45:39形式

            now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

            # 插入数据

            sql = "insert ignore   into topic(`title`,`topic_id`,`create_time`) values('{}','{}','{}')".format(title,

                                                                                                         topic_id, now)

            #print(sql)

            reCount = self.cur.execute(sql)

            self.conn.commit()

        self.cur.close()

        self.conn.close()

        return parent_topic

　　

child_topic.py

# *_*coding:utf-8 *_*

import json

import urllib

from time import sleep

import pymysql

from lxml import etree

import requests

class GetChildTopic(object):

    def __init__(self):

        self.conn = pymysql.connect(host='192.168.33.10', user='root', passwd='root', db='spider', charset='utf8')

        self.cur = self.conn.cursor()

    def sql_filter(self,sql, max_length=20):

        dirty_stuff = ["\"", "\\", "/", "*", "'", "=", "-", "#", ";", "<", ">", "+", "%", "$", "(", ")", "%", "@", "!"]

        for stuff in dirty_stuff:

            sql = sql.replace(stuff, "")

        return sql[:max_length]

    def get_child_data(self,parent_id, total_pages):

        int(parent_id)

        for page in range(1, total_pages + 1):

            #sleep(1)

            output = []

            print("now_parent_id",parent_id,"now_page:",page)

            url = "https://www.zhihu.com/node/TopicsPlazzaListV2"

            headers = {

                "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36",

            }

            offset = (page - 1) * 20

            data = {'method': 'next', "params": json.dumps({"topic_id": parent_id, "offset": offset, "hash_id": ""})}

            response = requests.post(url, data=data, headers=headers)

            print(url,response,);

            print(data)

            res = response.json()['msg']

            if(len(res) < 0):

                break;

            for item in res:

                html = etree.HTML(item)

                title = html.xpath('//img/@alt')[0]

                img_url = html.xpath('//img/@src')[0]

                topic_url = html.xpath('//a[1]/@href')[0]

                topic_id = topic_url.split('/')[-1]

                topic_url = urllib.parse.urljoin(url, topic_url)

                desc = html.xpath('//p/text()')

                if desc is not None and len(desc) == 1:

                    desc = desc[0]

                else:

                    desc = ''

                title = self.sql_filter(title, 200)

                img_url = self.sql_filter(img_url, 200)

                topic_url = self.sql_filter(topic_url, 200)

                desc = self.sql_filter(desc, 200)

                output.append({'title': title, 'img_url': img_url, "topic_url": topic_url, "desc": desc, "topic_id": topic_id,'parent_id': parent_id})

            print(output)

            self.save_child_topic(output)

    def save_child_topic(self,data):

        for item in data:

            import time

            # 格式化成2016-03-20 11:45:39形式

            now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())

            # 插入数据

            sql = "insert  ignore into topic(`title`,`topic_id`,`img_url`,`parent_id`,`desc`,`topic_url`,`level`,`create_time`) values('{}','{}','{}','{}','{}','{}','{}','{}')".format(

                item['title'], item['topic_id'], item['img_url'], item['parent_id'], item['desc'], item['topic_url'], 1,

                now)

            #print(sql)

            reCount = self.cur.execute(sql)

            self.conn.commit()

    def __del__(self):

        self.cur.close()

        self.conn.close()

　　sql

CREATE TABLE `topic` (

  `id` int(11) NOT NULL AUTO_INCREMENT,

  `title` varchar(255) NOT NULL DEFAULT '' COMMENT '标题',

  `topic_id` int(11) NOT NULL,

  `img_url` varchar(255) NOT NULL DEFAULT '' COMMENT '子标题图片',

  `parent_id` int(11) NOT NULL DEFAULT '0',

  `desc` text,

  `create_time` varchar(255) NOT NULL DEFAULT '',

  `topic_url` varchar(255) DEFAULT '' COMMENT '子标题超链接',

  `level` tinyint(4) NOT NULL DEFAULT '0' COMMENT '0父级 ',

  PRIMARY KEY (`id`),

  UNIQUE KEY `uni_top_par` (`topic_id`,`parent_id`),

  KEY `index_parent_id` (`parent_id`),

  KEY `index_topic_id` (`topic_id`)

) ENGINE=InnoDB AUTO_INCREMENT=8379 DEFAULT CHARSET=utf8mb4;