# -*- coding: utf-8 -*-
from urllib.parse import urlencode
import requests
import scrapy
import re
import json
from ..items import TmallItem
cookie = {'thw': 'cn', 'hng': 'CN%7Czh-CN%7CCNY%7C156', 'tracknick': 'yzhy1372', 'tg': '', 'miid': '', 'x': 'e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0%26__ll%3D-1%26_ato%3D0', '_cc_': 'UIHiLt3xSw%3D%3D', 'enc': '52fRsc7qpI96LDqf%2FkMA7AfWwN0%2BYmGMXsa4AdC3He4jEbrP%2BRbmYwz%2Bn3xwMrIk4fqBuRCR6BYtQvI%2FP7UBRw%3D%3D', 'UM_distinctid': '165c600d3903a8-0dc9190eb920d3-c343567-100200-165c600d39319', 'cna': 'iSbqEnsQrkoCAXM7KlL0pQWu', 't': '8489c373deedc2a297ebe4c4ad6debb5', '_uab_collina': '', '_umdata': '6AF5B463492A874D05644EF9A3CE888C0BB3EC8395620198BCCF71C40733CB6AAB98C444C566382ECD43AD3E795C914C010C8EDA083E64FAFA9E46E3CF4DEA41', '_m_h5_tk': 'bf46d22c8564ad537f01664eb002112c_1539921942514', '_m_h5_tk_enc': 'f2a1bff4b69d2c036314c66504744070', 'v': '', 'cookie2': '2b9488dea40dbe840f20ea5f14836ef7', '_tb_token_': 'fb83ee7ebeed7', 'alitrackid': 'www.taobao.com', 'lastalitrackid': 'www.taobao.com', 'JSESSIONID': '9787B4CF4D2812E2BA1E407B224AE53A', 'isg': 'BOfnzJhvcDexNPXcxwaGYkk8dhtxxJBNn5b9BrlUMnacqAVqyz-ynoHpzuiTQJPG', 'Hm_lvt_dde6ba2851f3db0ddc415ce0f895822e': '1539912803,1539913323,1539944839,1539944853', 'Hm_lpvt_dde6ba2851f3db0ddc415ce0f895822e': '', 'unb': '', 'uc1': 'cookie16=VFC%2FuZ9az08KUQ56dCrZDlbNdA%3D%3D&cookie21=WqG3DMC9FxUx&cookie15=VT5L2FSpMGV7TQ%3D%3D&existShop=false&pas=0&cookie14=UoTfItnW5e2f1g%3D%3D&tag=8&lng=zh_CN', 'sg': '', '_l_g_': 'Ug%3D%3D', 'skt': '5c93ad4f47f0c1ca', 'cookie1': 'U%2BTs5qAQHjB1CoYPMJcEQ4UfC6zh%2FdhqLG66mPjcz38%3D', 'csg': 'e312c3a6', 'uc3': 'vt3=F8dByRmq%2Bp63ob4wR7I%3D&id2=VW3j%2BbmcVcIV&nk2=GhETDBFSx%2Fs%3D&lg2=VT5L2FSpMGV7TQ%3D%3D', 'existShop': 'MTUzOTk0NTUzNw%3D%3D', 'lgc': 'yzhy1372', 'dnk': 'yzhy1372', '_nk_': 'yzhy1372', 'cookie17': 'VW3j%2BbmcVcIV', 'mt': 'np='}
class MianbaoSpider(scrapy.Spider):
name = "mianbao"
# allowed_domains = ["https://www.taobao.com"]
def start_requests(self):
url = 'https://s.taobao.com/search'
pars = {
'q': '女士上衣', #搜索关键字
'initiative_id': 'staobaoz_20181019',
'ie': 'utf8',
'tab': 'mall', #搜索天猫 1,all天猫淘宝 2，tmall天猫 3，old二手
# 's': '0', #页码 44递增
'sort': 'sale-desc' #默认 default
#排序类型
# #credit-desc信用排序
# #price-asc 价格升序
#price-desc 价格降序序
}
data = urlencode(pars)
urls = [url+'?'+data+'&s='+str(page) for page in range(0,450,44)] #翻页爬取
for u in urls:
yield scrapy.Request(u,self.mianbao,cookies=cookie)
def mianbao(self, response):
res = re.compile(r'g_page_config = (.*?);\s*g_srp_loadCss',re.S)
datas = json.loads(res.findall(response.text)[0])['mods']['itemlist']['data']['auctions']
for i in datas:
title = i['raw_title'] #商品名称
pic_url = 'http:'+i['pic_url'] #图片链接 #列表页图片
# view_price = i['view_price'] #商品价格
detail_url = 'https:'+i['detail_url'] #商品详情url
nick = i['nick'] #店铺名称
view_sales = i['view_sales'] #付款人数
item_loc = i['item_loc'] #商品所在地
comment_count = i['comment_count'] #评论数
user_id = i['user_id'] #取评论内容用
yield scrapy.Request(detail_url,self.detail_info,meta={'title':title,'nick':nick,'view_sales':view_sales,'item_loc':item_loc,'comment_count':comment_count,'pic_url':pic_url,'user_id':user_id})
def detail_info(self,response):
item = TmallItem()
res = re.compile(r'"defaultItemPrice":"(.*?)",',re.S)
price = res.findall(response.text)[0] #单价
good_imgs = response.xpath('//*[@id="J_UlThumb"]/li/a/img/@src').extract()#抓取图片
good_info = response.xpath('//*[@id="J_AttrUL"]/li/text()').extract()
if len(good_info) == 0: #商品详情
good_infos = '暂无'
else:
good_infos = good_info
item_id = re.findall(r'id=(.*?)&',response.url)[0] #这里是取出商品id
user_id = response.meta['user_id'] #取出商家id
url = 'https://rate.tmall.com/list_detail_rate.htm'
data = {
'itemId': item_id, #商品id
'sellerId': user_id #商家id
}
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36'
}
try:
rote_response = requests.get(url=url,params=data,headers=headers) #发起请求
rote_json = json.loads(re.findall(r'jsonp128\((.*?)\)',rote_response.text)[0])['rateDetail']['rateList']
rote_list = [] # 评论列表
for i in rote_json:
rote_dict = {}
rote_dict['auctionSku'] = i['auctionSku'] #购买商品名称
rote_dict['rateContent'] = i['rateContent'] #商品评论内容
rote_dict['pics'] = i['pics'] #评价图片
if len(rote_list) < 5: #每件商品只抓5条评论
rote_list.append(rote_dict) #把评论内容放到列表里
except:
print('该商品评论 无法抓取')
rote_list = []
item['title'] = response.meta['title']
item['nick'] = response.meta['nick']
item['price'] = price
item['view_sales'] = response.meta['view_sales']
item['item_loc'] = response.meta['item_loc']
item['comment_count'] = response.meta['comment_count']
item['pic_url'] = response.meta['pic_url']
item['good_infos'] = good_infos
item['good_imgs'] = good_imgs
item['rote_list'] = rote_list
return item
# -*- coding: utf-8 -*-
import pymongo
mongo = pymongo.MongoClient('127.0.0.1',27017)
mongodb = mongo['tmall']
mongocoll = mongodb['good_info']
import os
import requests
import csv
import pymysql
db = pymysql.connect(
db = 'test',
user = 'root',
port = 3306,
host = 'localhost',
password = 'mysql',
charset = 'utf8'
)
cursor = db.cursor()
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
class TmallPipeline(object):
def process_item(self, item, spider):
good_imgs = item['good_imgs']
title = item['title']
path = 'tmalls/' + title #商品信息路径
if not os.path.exists(path):
os.mkdir(path)
img = [] #更改图片链接
count = 0
for i in good_imgs:
count += 1
url = 'https:'+i[:-13]
img.append(url)
with open(path+'\\'+str(count)+'.jpg','wb') as f: #写入图片
response = requests.get(url)
f.write(response.content)
item['good_imgs'] = img
with open(path+'\\'+'商品信息'+'.csv','w+',encoding='utf-8',newline='') as f:
writer = csv.writer(f)
for k, j in dict(item).items():
datas = [
[k, j]
]
writer.writerows(datas)
writer.writerows('\n')
mongocoll.insert(dict(item))
title = item['title']
price = item['price']
good_infos = item['good_infos']
view_sales = item['view_sales']
comment_count = item['comment_count']
item_loc = item['item_loc']
nick = item['nick']
sql = 'insert into tmall values (0,%s,%s,%s,%s,%s,%s,%s)'
cursor.execute(sql,[title,price,str(good_infos),view_sales,comment_count,item_loc,nick])
db.commit()
return item