无法在webclient中完全提取数据

问题描述

我试图从该网站提取数据，首先我试图获取类别，并从每个类别我得到子类别，然后从每个子类别我试图阅读和提取一些文本。



我正在使用webclient我面临一些奇怪的问题，有时我不会读取数据有时会读取数据。



有时我会得到30个类别，但从10个类别获得子类别有时候我会得到10个类别并使用网络客户端读取所有子类别。



如何解决这个问题？

以下是代码：

Hi , im trying to extract data from that website , first im trying to get categories and from each categories i am getting subcategories and then from each subcategory im trying to read and extract some text.

Im using webclient that im facing some strange problem that sometimes i doesnt read data sometimes it reads data.

Sometimes i get 30 categories but get subcategories from only 10 sometimes i get 10 categories and read all subcategories using web client.

how to solve this problem ?
followin is he code :

public Extract(string url)
       {

           client = new WebClient();
           strm = client.OpenRead(url);
           strrdr = new StreamReader(strm, Encoding.ASCII);
           categorylines = new List<string>();
           subcategorylines = new List<string>[30];

           while (strrdr.Peek() > 0)
           {

               string line = strrdr.ReadLine();
               line = line.Replace("\n", String.Empty);
               line = line.Replace("\t", string.Empty);
               line = line.Replace("\r", string.Empty);
               line = line.Replace("\\", "");
               //System.Threading.Thread.Sleep(100);
               ExtractLines(line);

           }
           strrdr.Close();
       }

       public void ExtractSubcategories() {
           string url = null;
           string name = null;

           for (int i = 0; i < Categories.Category.Count; i++)
           {
               foreach (var item in subcategorylines[i])
               {
                   find1 = new Regex(@"href="".+"">", RegexOptions.IgnoreCase);
                   find2 = new Regex(@">.+<\/a>", RegexOptions.IgnoreCase);
                   m1 = find1.Match(item);
                   m2 = find2.Match(item);
                   if (m1.Success)
                   {
                       url = item.Substring(m1.Index+6, m1.Length - 8);
                       url = "www.codeproject.com" + url;
                   }
                   if (m2.Success)
                   {
                       name = item.Substring(m2.Index + 1, m2.Length - 5);

                   }
                   ArticleSubCategory sub = new ArticleSubCategory(name, url);
                   Categories.Category[i].SubCategories.Add(sub);
               }


           }
       }
       public void ExtractCategory() {
               string url = null;
               string name = null;
               tblcategories = new DataTable();
               foreach (var item in categorylines)
               {
                   url = GetLine(@"href="".+"">", 6, 8, item);
                   name = GetLine(@">.+<\/a>", 1, 5, item);

                   Categories.Add(new ArticleCategory(name, url));


               }
       }

       public string GetLine(string regex, int start, int end,string line)
       {
           find1 = new Regex(regex, RegexOptions.IgnoreCase);
           m1 = find1.Match(line);
           if (m1.Success)
           {
               return line.Substring(m1.Index+start,m1.Length-end);
           }
           else return null;
       }

       //public void ExtractArticle() { }

       public void ExtractLines(string line)
       {
           string a;

           if ((a = GetLine(categoryRegex, 0, 0, line)) != null)
           {
               tmp++;
               categorylines.Add(a);
               subcategorylines[tmp] = new List<string>();
           }
           if((a=GetLine(subcategoryRegex,0,0,line))!=null)
           {

               subcategorylines[tmp].Add(a);

           }
           //find1= new Regex(categoryRegex, RegexOptions.IgnoreCase);
           //find2= new Regex(subcategoryRegex, RegexOptions.IgnoreCase);
           //m1 = find1.Match(line);
           //m2= find2.Match(line);
           //if (m1.Success)
           //{
           //    tmp = tmp + 1;
           //    categorylines.Add(m1.Value);
           //    subcategorylines[tmp] = new List<string>();
           //}
           //else
           //{
           //    if (m2.Success)
           //    {
           //        subcategorylines[tmp].Add(m2.Value);
           //    }
           //}

       }

       public void ExtractArticleMeta() {

           tmp = 0;
           string a;
           foreach (var item in Categories.Category)
           {
               articleLines[tmp] = new List<string>();
               foreach (var subcat in item.SubCategories)
               {
                   client = new WebClient();
                   strm = client.OpenRead("http://"+subcat.Url);
                   strrdr = new StreamReader(strm, Encoding.ASCII);
                   while (strrdr.Peek()>0)
                   {
                       string line = strrdr.ReadLine();
                       line = line.Replace("\n", String.Empty);
                       line = line.Replace("\t", string.Empty);
                       line = line.Replace("\r", string.Empty);
                       line = line.Replace("\\", "");
                       if ((a= GetLine(articleregex,0,0,line))!=null)
                       {
                           articleLines[tmp].Add(a);
                       }

                   }


               }
               tmp = tmp + 1;
           }

       }

推荐答案

这篇关于无法在webclient中完全提取数据的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持！