配置scrapy-splash+python爬取医院信息（利用了scrapy-splash）

北京艾丽斯妇科医院（http://fuke.fuke120.com/）

首先先说一下配置splash

1.利用pip安装scrapy-splash库

pip install scrapy-splash

2.现在就要用到另一个神器（Docker）

Docker下载地址：https://www.docker.com/community-edition#/windows

3.安装好Docker后启动Docker拉取镜像

docker pull scrapinghub/splash

4.利用Docker运行splash

docker run -p 8050:8050 scrapinghub/splash（运行之后大家可以去浏览器输入http://192.168.99.100:8050检查Docker是否正确）

5settings.py配置

SPLASH_URL = 'http://192.168.99.100:8050'（重中之重，一个大坑，一定要注意这个IP就是192.168.99.100，我就一直用的自己IP一直没运行成功）

DOWNLOADER_MIDDLEWARES = {

'scrapy_splash.SplashCookiesMiddleware': 723,

'scrapy_splash.SplashMiddleware': 725,

'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,

}

SPIDER_MIDDLEWARES = {

'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,

}

DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'

HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

ROBOTSTXT_OBEY = True（此处注意，有的网站是True，而有的网站需要把它改成False）

　爬虫的py文件1.py

# -*- coding: utf-8 -*-

import re

from urllib.request import urlopen

from scrapy.http import Request

# from urllib.request import Request

from bs4 import BeautifulSoup

from lxml import etree

import pymongo

import scrapy

from scrapy.selector import HtmlXPathSelector

client = pymongo.MongoClient(host="127.0.0.1")

db = client.Health

collection = db.Healthclass  # 表名classification

import redis  # 导入redis数据库

r = redis.Redis(host='127.0.0.1', port=6379, db=0)

ii = 0

class healthcareClassSpider(scrapy.Spider):

    name = "HealthCare"

    allowed_domains = ["fuke120.com"]  # 允许访问的域

    start_urls = [

        "http://fuke.fuke120.com/",

    ]

    # 每爬完一个网页会回调parse方法

    def parse(self, response):

        global ii

        hxs = HtmlXPathSelector(response)

        hx = hxs.select('//div[@id="allsort"]/div[@class="item"]/span/a')

        hx1 = hxs.select('//div[@id="allsort"]/div[@class="item born"]/span/a')

        # hx2 = hxs.select('//div[@id="allsort"]/div[@class="item"]/div[@class="i-mc"]/div[@class="i-mc01"]/ul[@class="w_ul01"]/li/a')

        for secItem in hx:

            ii+=1

            url = secItem.select("@href").extract()

            c = "http://fuke.fuke120.com"+url[0]

            name = secItem.select("text()").extract()

            print(c)

            print(name)

            classid = collection.insert({'healthclass': name, 'pid': None})

            healthurl = '%s,%s,%s' % (classid, c, ii)

            r.lpush('healthclassurl',healthurl)

        for secItem1 in hx1:

            url = secItem1.select("@href").extract()

            c1 = "http://fuke.fuke120.com"+url[0]

            name1 = secItem1.select("text()").extract()

            print(c1)

            print(name1)

            classid = collection.insert({'healthclass': name1, 'pid': None})

            healthurl = '%s,%s,%s' % (classid, c1, 0)

            r.lpush('healthclassurl', healthurl)

　　2.py

# -*- coding: utf-8 -*-

import re

from urllib.request import urlopen

from urllib.request import Request

from bs4 import BeautifulSoup

from lxml import etree

import pymongo

import scrapy

from scrapy.selector import HtmlXPathSelector

from bson.objectid import ObjectId

# from scrapy.http import Request

# from urllib.request import urlopen

from scrapy.http import Request

# from hello.items import ZhaopinItem

# from scrapy.spiders import CrawlSpider, Rule

# from scrapy.linkextractors import LinkExtractor

from urllib.request import Request,ProxyHandler

from urllib.request import build_opener

client = pymongo.MongoClient(host="127.0.0.1")

db = client.Health            #库名dianping

collection = db.Diseaseclass          #表名classification

import redis        #导入redis数据库

r = redis.Redis(host='192.168.60.112', port=6379, db=0, charset='utf-8')

class healthcareClassSpider(scrapy.Spider):

    name = "HealthCare1"

    allowed_domains = ["fuke120.com"]  # 允许访问的域

    dict = {}

    start_urls = []

    def __init__(self):

        a = r.lrange('healthclassurl', 0,-1)

        for item in a:

            healthurl = bytes.decode(item)

            arr = healthurl.split(',')

            healthcareClassSpider.start_urls.append(arr[1])

            num = arr[2]

            pid = arr[0]

            url = arr[1]

            self.dict[url] = {"pid": pid, "num": num}

    def parse(self, response):

        nameInfo = self.dict[response.url]

        pid1 = nameInfo['pid']

        pid = ObjectId(pid1)

        num = nameInfo['num']

        hxs = HtmlXPathSelector(response)

        hx = hxs.select('//div[@class="x_con02_2"]/div[@class="x_con02_3"]/ul/li/p/a')

        for secItem in hx:

            url = secItem.select("@href").extract()

            url = "http://fuke.fuke120.com"+url[0]

            name = secItem.select("text()").extract()

            print(url)

            print(name)

            classid = collection.insert({'Diseaseclass': name, 'pid': pid})

            diseaseclassurl = '%s,%s,%s' % (classid, url, pid)

            r.lpush('diseaseclassurl', diseaseclassurl)

　　3.py

# -*- coding: utf-8 -*-

import re

from urllib.request import urlopen

from urllib.request import Request

from bs4 import BeautifulSoup

from lxml import etree

import pymongo

import scrapy

from scrapy_splash import SplashMiddleware

from scrapy.http import Request, HtmlResponse

from scrapy_splash import SplashRequest

from scrapy.selector import Selector

from scrapy.selector import HtmlXPathSelector

from bson.objectid import ObjectId

# from diseaseHealth.diseaseHealth.spiders.SpiderJsDynamic import phantomjs1

# from scrapy.http import Request

# from urllib.request import urlopen

from scrapy.http import Request

client = pymongo.MongoClient(host="127.0.0.1")

db = client.Health  # 库名dianping

collection = db.Treatclass  # 表名classification

#

import redis  # 导入redis数据库

#

r = redis.Redis(host='192.168.60.112', port=6379, db=0, charset='utf-8')

class healthcareClassSpider(scrapy.Spider):

    name = "HealthCare2"

    allowed_domains = ["fuke120.com"]  # 允许访问的域

    dict = {}

    start_urls = []

    def __init__(self):

        a = r.lrange('diseaseclassurl', 0,-1)

        for item in a:

            healthurl = bytes.decode(item)

            arr = healthurl.split(',')

            healthcareClassSpider.start_urls.append(arr[1])

            num = arr[2]

            pid = arr[0]

            url = arr[1]

            self.dict[url] = {"pid": pid, "num": num}

    def start_requests(self):

        for url in self.start_urls:

            yield SplashRequest(url, self.parse, args={'wait': 0.5})

    def parse(self, response):

            # a = response.body.decode('utf-8')

            # print(a)

        nameInfo = self.dict[response.url]

        pid1 = nameInfo['pid']

        pid = ObjectId(pid1)

        num = nameInfo['num']

        print(num)

        print(pid)

        hxs = HtmlXPathSelector(response)

        hx = hxs.select('//div[@class="dh01"]/ul[@class="ul_bg01"]/li/a')

        for secItem in hx:

            url = secItem.select("@href").extract()

            c = "http://fuke.fuke120.com" + url[0]

            name = secItem.select("text()").extract()

            print(c)

            print(name)

            classid = collection.insert({'Treatclass': name, 'pid': pid})

            treatclassurl = '%s,%s,%s' % (classid, c, pid)

            r.lpush('treatclassurl', treatclassurl)

　　大功告成，主要还是为了使用scrapy-splash。