DotnetSpider爬虫简单示例 net core

文章地址

https://blog.csdn.net/sD7O95O/article/details/78097556

　

安装爬虫框架  NUGET 安装DotnetSpider

创建HTTP协议数据包

            var site = new Site

            {

                CycleRetryTimes = 1,

                SleepTime = 200,

                Headers = new Dictionary<string, string>()

                {

                    {"Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8" },

                    {"Accept-Encoding","gzip, deflate, br" },

                    {"Accept-Language","zh-CN,zh;q=0.9" },

                    {"X-Requested-With","XMLHttpRequest" },

                    { "Referer", "https://blog.csdn.net/sD7O95O/article/details/78096027"},

                    { "Connection","keep-alive" },

                    { "Content-Type","text/html; charset=UTF-8" },

                    { "Host","blog.csdn.net"},

                    { "User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"}

                }

            };
            site.AddStartUrl(surl);

创建一个爬虫

            Spider spider = Spider.Create(site,             //

                new QueueDuplicateRemovedScheduler(),

                new CsdnArticleProcessor()                  //这个自定义的处理器

            ).AddPipeline(new CsdnArticlePipeline());       //自定义管道

　　

设置爬虫

　　　　　　  spider.Downloader = new HttpClientDownloader();

            spider.ThreadNum = 1;

            spider.EmptySleepTime = 3000;

　　

运行

 spider.Run();

　　

　

CsdnArticleProcessor 处理器  xpath 分析就不贴了

public class CsdnArticleProcessor : BasePageProcessor

    {

        protected override void Handle(Page page)

        {

            //page.Selectable.SelectList(Selectors.XPath("//table[@id='ip_list']/tr[2]/td[2]/text()")).Nodes();

            // 利用 Selectable 查询并构造自己想要的数据对象

            //xpath 得到自己需要的数据

　　　　　　　//...

　　　　　　　//page.AddResultItem("CountryResult", results);放入管道

        }

    }

    class CsdnArticlePipeline : BasePipeline

    {

        public override void Process(IEnumerable<ResultItems> resultItems, ISpider spider)

        {

　　　　　　var conlist = resukt.GetResultItem("CountryResult");获取数据
          //DB处理

        }

    }