java - HtmlUnit WebClient超时

在我之前有关HtmlUnit的问题中
Skip particular Javascript execution in HTML unit
和
Fetch Page source using HtmlUnit : URL got stuck

我曾经提到URL被卡住了。我还发现，由于HtmlUnit库中的一种方法(解析)没有执行失败，因此卡住了。

我对此做了进一步的工作。如果要花费超过指定的超时秒数，我将编写代码来退出该方法。

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;

public class HandleHtmlUnitTimeout {

public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException, InterruptedException, TimeoutException
    {
        Date start = new Date();
        String url = "http://ericaweiner.com/collections/";
        doWorkWithTimeout(url, 60);
    }

public static void doWorkWithTimeout(final String url, long timeoutSecs) throws InterruptedException, TimeoutException {
    //maintains a thread for executing the doWork method
    ExecutorService executor = Executors.newFixedThreadPool(1);
    //logger.info("Starting method with "+timeoutSecs+" seconds as timeout");
    //set the executor thread working

    final Future<?> future = executor.submit(new Runnable() {
        public void run()
            {
            try
                {
                getPageSource(url);
                }
            catch (Exception e)
                {
                throw new RuntimeException(e);
                }
        }
    });

    //check the outcome of the executor thread and limit the time allowed for it to complete
    try {
        future.get(timeoutSecs, TimeUnit.SECONDS);
    } catch (Exception e) {
        //ExecutionException: deliverer threw exception
        //TimeoutException: didn't complete within downloadTimeoutSecs
        //InterruptedException: the executor thread was interrupted

        //interrupts the worker thread if necessary
        future.cancel(true);

        //logger.warn("encountered problem while doing some work", e);
        throw new TimeoutException();
    }finally{
    executor.shutdownNow();
    }
}

public static void getPageSource(String productPageUrl)
    {
    try {
    if(productPageUrl == null)
        {
        productPageUrl = "http://ericaweiner.com/collections/";
        }

        WebClient wb = new WebClient(BrowserVersion.FIREFOX_3_6);
        wb.getOptions().setTimeout(120000);
        wb.getOptions().setJavaScriptEnabled(true);
        wb.getOptions().setThrowExceptionOnScriptError(true);
        wb.getOptions().setThrowExceptionOnFailingStatusCode(false);
        HtmlPage page = wb.getPage(productPageUrl);
        wb.waitForBackgroundJavaScript(4000);
        wb.closeAllWindows();
}
catch (FailingHttpStatusCodeException e)
    {
    e.printStackTrace();
    }
catch (MalformedURLException e)
    {
    e.printStackTrace();
    }
catch (IOException e)
    {
    e.printStackTrace();
    }
    }

}

这段代码确实来自doWorkWithTimeout(url，60);方法。但这不会终止。

当我尝试使用以下代码调用类似的实现时:

import java.util.Date;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.Logger;


public class HandleScraperTimeOut {

private static Logger logger = Logger.getLogger(HandleScraperTimeOut .class);


public void doWork() throws InterruptedException {
    logger.info(new Date()+ "Starting worker method ");
    Thread.sleep(20000);
    logger.info(new Date()+ "Ending worker method ");
    //perform some long running task here...
}

public void doWorkWithTimeout(int timeoutSecs) {
    //maintains a thread for executing the doWork method
    ExecutorService executor = Executors.newFixedThreadPool(1);
    logger.info("Starting method with "+timeoutSecs+" seconds as timeout");
    //set the executor thread working

    final Future<?> future = executor.submit(new Runnable() {
        public void run()
            {
            try
                {
                doWork();
                }
            catch (Exception e)
                {
                throw new RuntimeException(e);
                }
        }
    });

    //check the outcome of the executor thread and limit the time allowed for it to complete
    try {
        future.get(timeoutSecs, TimeUnit.SECONDS);
    } catch (Exception e) {
        //ExecutionException: deliverer threw exception
        //TimeoutException: didn't complete within downloadTimeoutSecs
        //InterruptedException: the executor thread was interrupted

        //interrupts the worker thread if necessary
        future.cancel(true);

        logger.warn("encountered problem while doing some work", e);
    }
    executor.shutdown();
}

public static void main(String a[])
    {
        HandleScraperTimeOut hcto = new HandleScraperTimeOut ();
        hcto.doWorkWithTimeout(30);

    }

}

如果有人可以看看并告诉我问题所在，那将非常有帮助。

有关问题的更多详细信息，可以查看Skip particular Javascript execution in HTML unit
和
Fetch Page source using HtmlUnit : URL got stuck

更新1
奇怪的是:future.cancel(true);在两种情况下都返回TRUE。
我的期望是:

对于HtmlUnit，由于进程仍在挂起，它应该返回FALSE。

具有正常的Thread.sleep();自该过程以来，它应该返回TRUE
被成功取消。

更新2
它仅与http://ericaweiner.com/collections/ URL一起挂起。如果我提供其他任何网址，即http://www.google.com，http://www.yahoo.com，则不会发送。在这些情况下，它将引发IntruptedException并退出流程。

看来http://ericaweiner.com/collections/页面源中包含某些会引起问题的元素。

最佳答案

Future.cancel(boolean)返回:

如果无法取消任务，则返回false，通常是因为它已经正常完成了

true否则

取消表示线程未在取消之前完成，取消标志设置为true，并且如果请求，则线程被中断。

中断线程menans叫做Thread.interrupt，仅此而已。 Future.cancel(boolean)不检查线程是否实际停止。

因此，在这种情况下，取消返回true是正确的。

中断线程意味着它应尽快停止，但不强制执行。您可以尝试使其停止/失败，以关闭其所需的资源或其他东西。我通常使用从套接字读取(等待传入数据)的线程来执行此操作。我关闭了 socket ，因此它不再等待。

关于java - HtmlUnit WebClient超时，我们在Stack Overflow上找到一个类似的问题：https://stackoverflow.com/questions/14559746/

Htmlunit

java - HtmlUnit WebClient超时