Miner.java 爬虫启动类

Miner.java 爬虫启动类

package com.iteye.injavawetrust.miner;

import java.util.concurrent.ThreadPoolExecutor;

import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;

/**

 * 爬虫启动类

 * @author InJavaWeTrust

 *

 */

public class Miner {

	private static final Log LOG = LogFactory.getLog(Miner.class);

	private MinerUrl minerUrl = null;

	private MinerConfig config = null;

	public Miner(MinerUrl minerUrl, MinerConfig config){

		this.minerUrl = minerUrl;

		this.config = config;

	}

	public void start(){

		//启动前 先校验配置

		if(!MinerUtil.checkBeforeStart(config)){

			LOG.info("配置出现问题，程序不能启动！！！");

			return;

		}

		MinerUtil.starTime = System.currentTimeMillis();

		//程序启动，将第一个起始页面放入待访问队列。

		MinerQueue.addUnVisited(minerUrl);

		//将URL 添加到URL队列 保证每个URL只访问一次

		MinerQueue.addUrlSet(minerUrl.getUrl());

		ThreadPoolExecutor executor = MinerThreadPool.getInstance();

		//download

		for(int i = 0; i < config.getMinerHtmlThreadNum(); i++){

			MinerHtmlThread minerHtml = new MinerHtmlThread(config);

			executor.execute(minerHtml);

		}

		//misering

		for(int i = 0; i < config.getMiseringThreadNum(); i++){

			MiseringThread misering = new MiseringThread(config);

			executor.execute(misering);

		}

		//store

		for(int i = 0; i < config.getMinserStoreThreadNum(); i++){

			MinerStoreThread minerStoreThread = new MinerStoreThread(config);

			executor.execute(minerStoreThread);

		}

		//monitor

		MinerMonitorThread minerMonitor = new MinerMonitorThread();

		executor.execute(minerMonitor);

	}

}

返回列表