给基友下载漫画看
代码：
# !usr/bin/python3.4
# -*- coding:utf-8 -*- import requests
import os
import time
import re
from lxml import etree
import random def geturl(url,postdata):
header = {'User-Agent':
'Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5',
'Referer':'http://m.1kkk.com/vol1-6871/',
'Host': 'manhua1023.61-174-50-131.cdndm5.com',
'Accept': 'image/png,image/*;q=0.8,*/*;q=0.5',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3',
'Connection': 'keep-alive',
} s = requests.Session()
r = s.post('http://m.1kkk.com/userdata.ashx',data = postdata)
_cookies = r.cookies
#print(r.content)
rs = s.get(url, headers=header,cookies = _cookies)
return rs def get(url):
header = {'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0',
'Referer': 'http://www.1kkk.com/manhua589/',
'Host': 'www.1kkk.com'} # 解析网页
html_bytes = requests.get(url, headers=header) return html_bytes def mget(url):
header = {'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0',
'Referer': 'http://m.1kkk.com/manhua589/',
'Host': 'm.1kkk.com'} # 解析网页
html_bytes = requests.get(url, headers=header) return html_bytes # 去除标题中的非法字符 (Windows)
def validateTitle(title):
# '/\:*?"<>|'
rstr = r"[\/\\\:\*\?\"\<\>\|]"
new_title = re.sub(rstr, "", title)
return new_title def prints(timesleep):
print('暂停' + str(timesleep) + '秒后开始批量下载图片，请保持网络畅通...')
time.sleep(timesleep) # 解析js
def regnext(js):
reg = r'(var.+?.split)'
all = re.compile(reg);
alllist = re.findall(all, js)
return alllist # 递归创建文件夹
def createjia(path):
try:
os.makedirs(path)
except:
print('目录已经存在：' + path) if __name__ == '__main__': html = get('http://www.1kkk.com/manhua589/').content.decode('utf-8', 'ignore') page = etree.HTML(html.lower())
# 得到网址后缀
hrefs = page.xpath('//ul[@class="sy_nr1 cplist_ullg"][2]/li/a/@href')
# 得到编号
hrefnames = page.xpath('//ul[@class="sy_nr1 cplist_ullg"][2]/li/a/text()')
# 得到页数
hrefpages = page.xpath('//ul[@class="sy_nr1 cplist_ullg"][2]/li/text()') href = []
hrefname = []
hrefpage = []
number = 1 # 不知道里面那几卷是不是漫画里面的
# 先抓下来再说
# 得到网址后缀
for temp in hrefs:
towurl = temp
href.append(towurl)
# 得到编号
for temp in hrefnames:
hrefname.append(temp)
# 得到页数
for temp in hrefpages:
hrefpage.append(temp.replace("页", "")) j = 0
filenamep = '../data/' + str(hrefname[0]) + "/"
createjia(filenamep) for i in range(0, len(href)):
for j in range(len(hrefpage)): # 6871、6872。。
hrefnumber = str(href[i]).replace("ch54-","").replace("/","").replace("vol1-","")
#print(hrefnumber)
# 构造jsurl
# 得到
# http://www.1kkk.com/vol1-6871/imagefun.ashx?cid=6871&page=1&key=65abd421f4aed565&maxcount=10
jsurl = "http://www.1kkk.com" + str(href[i]) + "/imagefun.ashx?cid=" + str(hrefnumber) + "&page=" + str(j + 1) + "&key=65abd421f4aed565&maxcount=10"
print(jsurl) # 构造image网址
html = get(jsurl).content.decode('utf-8', 'ignore')
html1 = regnext(html)
html1 = html1[0].replace("'.split", "").split('|') # http://manhua1023.61-174-50-131.cdndm5.com/1/589/6871/102_9224.jpg?cid=6871&key=d8ce90e0b3f013f292ef77e84da88990&type=1
image_1url = "http://manhua1023." + str(html1[19]) + "-" + str(html1[18]) + "-" + str(html1[9]) + "-" + str(
html1[10]) + ".cdndm5.com/1/589/" + str(href[i]) + "/" + str(html1[20]) + "?cid=" + str(6871) + "&key=" + str(
html1[8]) + "&type=1"
print(image_1url) # 构造image网址
filess = open(filenamep + str(j + 1) + '.jpg', 'wb') # 伪装posrdata
postdata = {
'cid': 6871,
'language': 1,
'mid': 589,
'page': j + 1,
'tp': 8,
'uid': 0
} # 即使正确的网址也是不能下载
pic = geturl(image_1url,postdata)
filess.write(pic.content)
filess.close()
print('已经写入第' + str(j + 1) + '张图片')
j = j + 1 # 每一次下载都暂停1-3秒
loadimg = random.randint(1, 3)
print('暂停' + str(loadimg) + '秒')
time.sleep(loadimg)
selenium抓取：
#!/usr/bin/python3.4
# -*- coding: utf-8 -*- from selenium import webdriver
import time
from selenium.webdriver.common.keys import Keys
import re # 去除标题中的非法字符 (Windows)
def validateTitle(title):
rstr = r"[\/\\\:\*\?\"\<\>\|]" # '/\:*?"<>|'
new_title = re.sub(rstr, "", title)
return new_title def getimg():
# http://www.cnblogs.com/fnng/p/3238685.html
# 打开火狐浏览器
# browser = webdriver.Chrome()
browser = webdriver.Firefox() # 设置浏览器大小
browser.set_window_size(1200, 900)
# 输入网址
browser.get("http://m.1kkk.com/vol1-6871/")
# 根据各自网速来判断网址加载时间
time.sleep(10) for i in range(10000): # 关掉广告
browser.find_element_by_class_name("ad_cross").click() # 翻页到最后面
browser.execute_script("""
(function () {
var y = 0;
var step = 100;
window.scroll(0, 0); function f() {
if (y < document.body.scrollHeight) {
y += step;
window.scroll(0, y);
setTimeout(f, 100);
} else {
window.scroll(0, 0);
document.title += "scroll-done";
}
} setTimeout(f, 1000);
})();
""")
print("下拉中...")
#time.sleep(180)
while True:
if "scroll-done" in browser.title:
break
else:
print("还没有拉到最底端...")
time.sleep(10) # while True:
# # 判断是否存在这个东西
# select = browser.find_element_by_xpath('//a[@class="readTipForm"]')
# if select:
# break
# else:
# print("还没有拉到最底端...")
# time.sleep(60) print("正在下载图片中...")
# 图片的命名
name = validateTitle(browser.current_url)
print("正在截图...")
time.sleep(5) # 截图
browser.save_screenshot("../jpg/cartoon/" + str(i + 1) + str(name) + ".png")
time.sleep(5) # 点击阅读下一章
browser.find_element_by_class_name("readTipForm").click()
print("准备进入下一章...")
time.sleep(5) browser.quit() if __name__ == '__main__':
getimg()