* 建站数据SuperSpider（简书）
* 本项目目的：
* 为练习web开发提供相关的数据；
* 主要数据包括：
* 简书热门专题模块信息、对应模块下的热门文章、
* 文章的详细信息、作者信息、
* 评论区详细信息、评论者信息等...
* 最后存储mysql数据库.
想学习爬虫的同学也可以瞧瞧
整个项目跑完花了近十个小时， 足见数据之多， 个人web开发练习用来充当建站数据也是绰绰有余的(～￣▽￣)～
代码注释写的挺详细的，我就直接上代码了。
主要代码：
/**
* 此类对简书文章内容页进行了详细的解析爬取；
* 封装后的唯一入口函数 {@link ArticleSpider#run(String, int)}.
*
* author As_
* date 2018-08-21 12:34:23
* github https://github.com/apknet
*/ public class ArticleSpider { /** 长度为4 ：阅读量、评论数、点赞数、文章对应的评论id */
private List<Integer> readComLikeList = new ArrayList<>(); /** 文章Id */
private long articleId; /**
* 此类的入口函数;
* 爬虫范围包括文章的详情信息、评论的详细信息；
* 并同时持久化数据；
* key实例：443219930c5b
*
* @param key
* @param flag_type 文章所属分类的索引
*/ public void run(String key, int flag_type){ try {
articleSpider(key, flag_type); // 以下参数由articleSpeder()函数获得
String comUrl = String.format("https://www.jianshu.com/notes/%d/comments?comment_id=&author_only=false&since_id=0&max_id=1586510606000&order_by=desc&page=1", readComLikeList.get(3)); comSpider(comUrl); } catch (Exception e) {
e.printStackTrace();
}
} /**
* 链接实例 https://www.jianshu.com/p/443219930c5b
* 故只接受最后一段关键字
* @param key
* @param flag_type 文章所属分类的索引
* @throws Exception
*/
private void articleSpider(String key, int flag_type) throws Exception { String url = "https://www.jianshu.com/p/" + key; Document document = getDocument(url); IdUtils idUtils = new IdUtils(); List<Integer> list = getReadComLikeList(document); articleId = idUtils.genArticleId();
String title = getTitle(document);
String time = getTime(document);
String imgCover = getImgCover(document);
String brief = getBrief(document);
String content = getContent(document);
String author = idUtils.genUserId();
int type = flag_type;
int readNum = list.get(0);
int comNum = list.get(1);
int likeNum = list.get(2); // 数据库添加对应对象数据
Article article = new Article(articleId, title, time, imgCover, brief, content, author, type, readNum, comNum, likeNum);
new ArticleDao().addArticle(article); User user = new User();
user.setUser(idUtils.genUserId());
user.setName(getAuthorName(document));
user.setImgAvatar(getAuthorAvatar(document));
new UserDao().addUser(user); } private Document getDocument(String url) throws IOException { Document document = Jsoup.connect(url)
.header("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:61.0) Gecko/20100101 Firefox/61.0")
.header("Content-Type", "application/json; charset=utf-8")
.header("Cookie", "_m7e_session=2e930226548924f569cb27ba833346db;locale=zh-CN;default_font=font2;read_mode=day")
.get();
return document;
} private String getTitle(Document doc){
return doc.select("h1[class=title]").text();
} private String getTime(Document doc){
Element element = doc.select("span[class=publish-time]").first();
return element.text();
} private String getImgCover(Document doc){
Element element = doc.select("img[data-original-format=image/jpeg]").first();
if (element.hasAttr("data-original-src")){
String url = element.attr("data-original-src").trim();
return SpiderUtils.handleUrl(url);
}
return null;
} private String getBrief(Document doc){
return doc.select("meta[name=description]").first().attr("content");
} private String getContent(Document doc){
Element element = doc.select("div[class=show-content-free]").first();
// System.out.println(element.html()); Elements eles = element.select("div[class=image-package]");
for(Element ele: eles){
Element imgEle = ele.select("img").first();
ele.replaceWith(imgEle);
} String result = element.html().replaceAll("data-original-", "");
return result;
} private String getAuthorAvatar(Document doc){
String url = doc.select("div[class=author]").select("img").attr("src");
return SpiderUtils.handleUrl(url);
} private String getAuthorName(Document doc){
Element element = doc.select("script[data-name=page-data]").first();
JSONObject jsonObject = new JSONObject(element.html()).getJSONObject("note").getJSONObject("author");
return jsonObject.getString("nickname");
} private List<Integer> getReadComLikeList(Document doc){
Element element = doc.select("script[data-name=page-data]").first();
JSONObject jsonObject = new JSONObject(element.html()).getJSONObject("note"); readComLikeList.add(jsonObject.getInt("views_count"));
readComLikeList.add(jsonObject.getInt("comments_count"));
readComLikeList.add(jsonObject.getInt("likes_count"));
readComLikeList.add(jsonObject.getInt("id")); System.out.println(Arrays.toString(readComLikeList.toArray()));
return readComLikeList;
} /**
* 评论区爬虫---包括：
* 评论楼层、时间、内容、评论者、评论回复信息等
* 具体可查看{@link Comment}
*
* @param url
* @throws Exception
*/
private void comSpider(String url) throws Exception {
URLConnection connection = new URL(url).openConnection(); // 需加上Accept header，不然导致406
connection.setRequestProperty("Accept", "*/*"); BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); String line;
StringBuilder str = new StringBuilder(); while ((line = reader.readLine()) != null) {
str.append(line);
} System.out.println(str.toString()); JSONObject jb = new JSONObject(str.toString()); boolean hasComment = jb.getBoolean("comment_exist");
if(hasComment){
int pages = jb.getInt("total_pages");
if(pages > 20){
// 某些热门文章评论成千上万， 此时没必要全部爬取
pages = 20;
}
int curPage = jb.getInt("page"); JSONArray jsonArray = jb.getJSONArray("comments");
Iterator iterator = jsonArray.iterator();
while(iterator.hasNext()){
JSONObject comment = (JSONObject) iterator.next();
JSONArray comChildren = comment.getJSONArray("children");
JSONObject userObj = comment.getJSONObject("user");
IdUtils idUtils = new IdUtils();
CommentDao commentDao = new CommentDao();
UserDao userDao = new UserDao(); int commentId = idUtils.genCommentId();
String userId = idUtils.genUserId(); System.out.println(comment.toString());
// 评论内容相关
String content = comment.getString("compiled_content");
String time = comment.getString("created_at");
int floor = comment.getInt("floor");
int likeNum = comment.getInt("likes_count");
int parentId = 0; // 评论者信息
String name = userObj.getString("nickname");
String avatar = userObj.getString("avatar"); Comment newCom = new Comment(commentId, articleId, content, time, floor, likeNum, parentId, userId, avatar, name);
commentDao.addComment(newCom); User user = new User();
user.setUser(userId);
user.setName(name);
user.setImgAvatar(avatar); userDao.addUser(user); // 爬取评论中的回复
Iterator childrenIte = comChildren.iterator();
while(childrenIte.hasNext()){
JSONObject child = (JSONObject) childrenIte.next(); Comment childCom = new Comment();
childCom.setId(idUtils.genCommentId());
childCom.setArticleId(articleId);
childCom.setComment(child.getString("compiled_content"));
childCom.setTime(child.getString("created_at"));
childCom.setParentId(child.getInt("parent_id"));
childCom.setFloor(floor);
childCom.setNameAuthor(child.getJSONObject("user").getString("nickname")); commentDao.addComment(childCom); }
} // 实现自动翻页
if(curPage == 1){
for(int i = 2; i <= pages; i++){
System.out.println("page-------------------> " + i);
int index = url.indexOf("page=");
String sub_url = url.substring(0, index + 5);
String nextPage = sub_url + i;
comSpider(nextPage);
}
}
}
}
}
and：
/**
* 模块(简书的专题)爬虫
* 入口：https://www.jianshu.com/recommendations/collections
* 模块实例： https://www.jianshu.com/c/V2CqjW
* 文章实例： https://www.jianshu.com/p/443219930c5b
* 故此爬虫多处直接传递链接后缀的关键字符串(如‘V2CqjW’、‘443219930c5b’)
*
* @author As_
* @date 2018-08-21 12:31:35
* @github https://github.com/apknet
*
*/ public class ModuleSpider { public void run() {
try { List<String> moduleList = getModuleList("https://www.jianshu.com/recommendations/collections"); for (String key: moduleList) {
// 每个Module爬取10页内容的文章
System.out.println((moduleList.indexOf(key) + 1) + "." + key);
for (int page = 1; page < 11; page++) {
String moduleUrl = String.format("https://www.jianshu.com/c/%s?order_by=top&page=%d", key, page); List<String> articleList = getArticleList(moduleUrl); for (String articlekey: articleList) {
new ArticleSpider().run(articlekey, moduleList.indexOf(key) + 1);
}
}
} } catch (Exception e) {
e.printStackTrace();
}
} /**
* 返回Module 关键字符段
* @param url
* @return
* @throws IOException
* @throws SQLException
*/ private List<String> getModuleList(String url) throws IOException, SQLException { List<String> moduleList = new ArrayList<>();
int i = 0; Document document = Jsoup.connect(url).get();
Element container = document.selectFirst("div[id=list-container]");
Elements modules = container.children();
for(Element ele: modules){
String relUrl = ele.select("a[target=_blank]").attr("href");
String moduleKey = relUrl.substring(relUrl.indexOf("/c/") + 3);
moduleList.add(moduleKey); // System.out.println(moduleKey);
// 以下爬取数据创建Module对象并持久化 String name = ele.select("h4[class=name]").text();
String brief = ele.selectFirst("p[class=collection-description]").text();
String imgcover = SpiderUtils.handleUrl(ele.selectFirst("img[class=avatar-collection]").attr("src")); String articleNu = ele.select("a[target=_blank]").get(1).text();
int articleNum = Integer.parseInt(articleNu.substring(0, articleNu.indexOf("篇"))); String userNu = ele.select("div[class=count]").text().replace(articleNu, "");
Matcher matcher = Pattern.compile("(\\D*)(\\d*\\.\\d*)?(\\D*)").matcher(userNu);
matcher.find();
int userNum = (int)(Float.parseFloat(matcher.group(2)) * 1000); Module module = new Module((++i), name, brief, imgcover, userNum, articleNum, "apknet");
new ModuleDao().addModule(module); System.out.println(name + "------------------>");
}
return moduleList;
} /**
* 文章链接实例 https://www.jianshu.com/p/443219930c5b
* 故此处返回List的String为url后的那段字符串
*
* @param url
* @return
*/
private List<String> getArticleList(String url) { System.out.println("getArticleList: --------------------->"); List<String> keyList = new ArrayList<>(); Document document = null;
try {
document = Jsoup.connect(url).get();
} catch (Exception e) {
e.printStackTrace();
}
Element ulElement = document.select("ul[class=note-list]").first(); Elements elements = ulElement.select("li[id]");
for (Element ele : elements) {
String relUrl = ele.select("a[class=title]").attr("href");
String key = relUrl.substring(relUrl.indexOf("/p/") + 3);
keyList.add(key);
System.out.println(key);
} return keyList;
} }
最后附上实验截图：
完整项目地址：