package test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import javax.swing.JOptionPane;
import org.openqa.selenium.By;
import org.openqa.selenium.JavascriptExecutor;
import org.openqa.selenium.Keys;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxBinary;
import org.openqa.selenium.firefox.FirefoxDriver;
import org.openqa.selenium.firefox.FirefoxOptions;
import org.openqa.selenium.interactions.Actions;
public class SuperLOFTERDownloader7 {
// static String html;
static int urlCount = 0;
static String username = "fenchenyue";
static String url = "http://" + username + ".lofter.com/view";
static String picUrl;
static boolean ifToBreak = false;
static ArrayList<String> urlList = new ArrayList<String>();
// 注意:目录路径必须以"\"结尾
static String downloadToDir = "E:\\爬虫\\妹纸2\\";
public static void main(String[] args) {
System.setProperty("webdriver.gecko.driver", "C:\\Users\\Jim\\Desktop\\GeckoDriver\\geckodriver.exe");
FirefoxOptions options = new FirefoxOptions();
// 启动配置"不加载图片"
options.addPreference("permissions.default.image", 2);
// 启动参数"无界面"
FirefoxBinary myBinary = new FirefoxBinary();
myBinary.addCommandLineOptions("--headless");
options.setBinary(myBinary);
FirefoxDriver driver = new FirefoxDriver(options);
driver.manage().timeouts().implicitlyWait(16, TimeUnit.SECONDS); // 隐式等待
driver.get(url);
// ((JavascriptExecutor)
// driver).executeScript("document.querySelector('.m-txtsch').style.display=\"inline\"");
int givenNumber = Integer.parseInt(driver.findElementByCssSelector(
"body > div.g-bdfull.g-bdfull-show.ztag > div.g-bdc.ztag > div.m-fbar.f-cb > div.schbtn.f-cb > div:nth-child(1) > div > div.txt > a.ztag.currt > span")
.getAttribute("innerHTML"));
Actions action = new Actions(driver);
Timer timer = new Timer();
timer.schedule(new TimerTask() {
@Override
public void run() {
// TODO Auto-generated method stub
action.sendKeys(/* driver.findElement(By.cssSelector("body")), */ Keys.END).perform();
}
}, 1, 700);
/*
* WebDriverWait wait = new WebDriverWait(driver, 256, 2048);
* wait.until(ExpectedConditions .numberOfElementsToBe(By.
* cssSelector("div.ztag > div.m-filecnt.m-filecnt-1 > ul > li"), number));
*/
new Thread(new Runnable() {
@Override
public void run() {
// TODO Auto-generated method stub
int flag = JOptionPane.showConfirmDialog(null, "正在收集资源...\n如果觉得时间过长,点击'是'提前中断", "是否中断",
JOptionPane.YES_NO_OPTION);
if (flag == JOptionPane.YES_OPTION) {
ifToBreak = true;
}
}
}).start();
// 周期查询文档是否加载完或者用户选择中断
String js = "let count=0;document.querySelectorAll('.g-bdc > div:nth-child(3) > div.m-filecnt.m-filecnt-1 > ul').forEach(function(e,i){count+=e.children.length});return count;";
while (true) {
long countFromJs = (long) ((JavascriptExecutor) driver).executeScript(js);
System.out.println("已收集到资源数 : " + countFromJs);
if (Math.abs((int) countFromJs - givenNumber) < 5 || ifToBreak) {
break;
}
try {
Thread.sleep(1600);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
timer.cancel();
for (WebElement element : driver.findElements(
By.cssSelector("div.ztag > div.m-filecnt.m-filecnt-1 > ul > li > a > div > div > img.realimg"))) {
picUrl = element.getAttribute("src").split("\\?")[0];
System.out.println(picUrl);
urlList.add(picUrl);
urlCount++;
}
System.out.println("爬取到" + givenNumber + "篇文章中的" + urlCount + "张图的url");
int flag = JOptionPane.showConfirmDialog(null, "核对url,是否开始下载?", "是否继续", JOptionPane.YES_NO_OPTION);
if (flag == JOptionPane.NO_OPTION) {
System.exit(0);
} else if (flag == JOptionPane.YES_OPTION) {
System.out.println("正在下载.....");
try {
Runtime.getRuntime().exec("explorer " + downloadToDir);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("完成,成功下载了" + DownloadFromUrlList3.download(urlList, downloadToDir) + "张图");
System.out.println("失败数:" + DownloadFromUrlList3.errCount);
}
// JOptionPane.showMessageDialog(null, "浏览器可以关闭了吗?");
driver.quit();
System.exit(0);
}
}
package test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
public class DownloadFromUrlList3 {
static private int count = 0;
static ThreadGroup myThreadGroup = new ThreadGroup("myGroup");
static picThread myThread;
public static int errCount = 0;
static long timeOut = 8 * 1000;
public static synchronized int updateCount() {
count++;
return count;
};
public static int download(ArrayList<String> urlList, String dirPath) {
if (new File(dirPath).exists()) {
if (!new File(dirPath).isDirectory()) {
System.out.println("ERROR!! THE PATH GIVEN ISN'T A DIRECTORY");
return 0;
}
} else {
new File(dirPath).mkdir();
}
for (String urlstr : urlList) {
String myUrlStr = urlstr;
myThread = new picThread(myThreadGroup, new Runnable() {
@Override
public void run() {
// TODO Auto-generated method stub
try {
URL url = new URL(myUrlStr);
BufferedInputStream is;
try {
// 网络流量一定要用高效的buffered
is = new BufferedInputStream(url.openStream());
} catch (Exception e) {
// TODO Auto-generated catch block
System.out.println("发现一个url资源出错:\n" + myUrlStr);
errCount++;
return;
}
String extention = "." + HttpURLConnection.guessContentTypeFromStream(is).split("/")[1];
File file = new File(dirPath + updateCount() + extention);
((picThread) Thread.currentThread()).file = file;
file.createNewFile();
OutputStream os = new FileOutputStream(file);
int len;
byte[] buffer = new byte[1024];
while ((len = is.read(buffer)) != -1) {
os.write(buffer, 0, len);
}
is.close();
os.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
});
myThread.urlStr = myUrlStr;
myThread.start();
// threadList.add(myThread);
try {
myThread.join(timeOut);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/*
* for (Thread iThread : threadList) { try { // 等待所有线程执行完毕 iThread.join(1000 *
* 15); } catch (InterruptedException e) { e.printStackTrace(); } }
*/
try {
Thread.sleep(1024);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
Thread[] activeThreads = new Thread[myThreadGroup.activeCount()];
myThreadGroup.enumerate(activeThreads);
for (Thread thread : activeThreads) {
System.out.println("一张图片下载超时 :" + ((picThread) thread).file.getName() + "\n" + ((picThread) thread).urlStr);
errCount++;
// ((picThread) thread).file.delete();
}
return count - myThreadGroup.activeCount();
}
// 使用示例:
public static void main(String[] args) {
ArrayList<String> list = new ArrayList<String>();
list.add(
"http://imglf1.nosdn.127.net/img/WnhZUEZYSVNPd1lsd1doNjVPSWVkREFwNTRrNTdUM2tQYkk2bHVabHpIYjJOZVFFeEdaUWRnPT0.jpg");
list.add("http://imgsrc.baidu.com/imgad/pic/item/b58f8c5494eef01f2c2e59feebfe9925bc317dd6.jpg");
list.add(
"http://imglf1.nosdn.127.net/img/WnhZUEZYSVNPd1p2Wk54NlRNMTZKLzJnci9HanJsbUNFUTlJdWdFaDhJQUlQQ3h5Y1kzOFlRPT0.jpg");
list.add("http://b.hiphotos.baidu.com/image/pic/item/7a899e510fb30f247b237cc9c195d143ac4b03ba.jpg");
System.out.println("完成,成功下载了:" + download(list, "C:\\Users\\Jim\\Desktop\\testPic\\"));
System.out.println("失败:" + DownloadFromUrlList3.errCount);
System.exit(0);
}
}
class picThread extends Thread {
public String urlStr = "";
public File file;
/*
* public void setUrl(String url) { urlStr = url; }
*
* public String getUrl() { return urlStr; }
*/
public picThread(ThreadGroup group, Runnable run) {
super(group, run);
}
}