JAVA编程131——httpclient+jsoup爬一篇小说

一、maven坐标

<dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
        <dependency>
            <groupId>org.apache.httpcomponents</groupId>
            <artifactId>httpclient</artifactId>
            <version>4.5.2</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.25</version>
            <!--<scope>test</scope>-->
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.10.2</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/junit/junit -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>

二、demo

package com.mollen;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import java.nio.charset.Charset;

/**
 * @ClassName: demo
 * @Auther: Mollen
 * @CreateTime: 2019-01-15  19:27:20
 * @Description:
 */
public class demo {
       @Test
        public void getContent() throws Exception {
            String url = "https://read.qidian.com/chapter/_h17RCSkeXScikCo3ZPkrg2/_LrKPdZirvtMs5iq0oQwLQ2";
             while(url!=null){
                 //1.发送一个请求
                 HttpGet httpGet = new HttpGet(url);
                 CloseableHttpClient httpClient = HttpClients.createDefault();
                 //2.获得响应
                 CloseableHttpResponse response = httpClient.execute(httpGet);
                 HttpEntity entity = response.getEntity();
                 String html = EntityUtils.toString(entity, Charset.forName("utf-8"));
                 //3.解析章节内容
                 Document doc = Jsoup.parse(html);
                 //3.1 获取章节名称
                 Element name = doc.select(".j_chapterName").get(0);
                 System.out.println("------------------------------------------------------------------------");
                 System.out.println(name.text());
                 //3.2 获取章节内容
                 Elements contents = doc.select("div[class=read-content j_readContent] p");
                 for (Element content : contents) {
                     System.out.println(content.text());
                 }
                 //3.3 获取下一个章节的url
                 Elements nextUrls = doc.select("#j_chapterNext");
                 Element nextUrl = nextUrls.get(0);
                 url = "https:"+nextUrl.attr("href");
             }
        }
}