|
|
@@ -41,46 +41,56 @@ namespace FCS.Crawler.ZCLotteryTeam
|
|
|
var ds = services.Query<F_Events>("and Remark is not null").ToList();
|
|
|
if (ds != null && ds.Count > 0)
|
|
|
{
|
|
|
- int max = ds.Count;
|
|
|
- int num = 0;
|
|
|
- ds.ForEach(async p =>
|
|
|
- {
|
|
|
- var url = p.Remark.ToString();
|
|
|
- var eventId = p.Id.ToString();
|
|
|
- var eventName = p.Name.ToString();
|
|
|
-
|
|
|
- await Task.Run(() =>
|
|
|
- {
|
|
|
- var mainUrl = "http://saishi.zgzcw.com" + url;
|
|
|
-
|
|
|
- HtmlDocument doc = CommonHelper.GetHtml(mainUrl, new Dictionary<string, string>(), "足彩", "", 10000, 100);
|
|
|
- var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='select_options']/a");
|
|
|
- //循环赛季
|
|
|
- if (TongJiListData != null && !doc.DocumentNode.InnerText.Contains("球会友谊"))
|
|
|
- {
|
|
|
- foreach (var item in TongJiListData)
|
|
|
+ //int max = ds.Count;
|
|
|
+ //int num = 0;
|
|
|
+ ds.ForEach(p =>
|
|
|
+ {
|
|
|
+ var url = p.Remark.ToString();
|
|
|
+ var eventId = p.Id.ToString();
|
|
|
+ var eventName = p.Name.ToString();
|
|
|
+
|
|
|
+ Task.Run(() =>
|
|
|
+ {
|
|
|
+ var mainUrl = "http://saishi.zgzcw.com" + url;
|
|
|
+
|
|
|
+ HtmlDocument doc = CommonHelper.GetHtml(mainUrl, new Dictionary<string, string>(), "足彩", "", 10000, 100);
|
|
|
+ var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='select_options']/a");
|
|
|
+ //max += TongJiListData.Count();
|
|
|
+ //循环赛季
|
|
|
+ if (TongJiListData != null && !doc.DocumentNode.InnerText.Contains("球会友谊"))
|
|
|
+ {
|
|
|
+ Task.Run(() =>
|
|
|
{
|
|
|
- var sjurl = item.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
|
|
|
- sjurl = sjurl.Replace("－", "-");
|
|
|
- GetEventHtml(sjurl, eventId, eventName, doc.DocumentNode.InnerText.Contains("球会友谊"));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- });
|
|
|
- lock (g)
|
|
|
- {
|
|
|
- num++;
|
|
|
- Monitor.Pulse(g); //完成，通知等待队列,告知已完，执行下一个。
|
|
|
- }
|
|
|
+ foreach (var item in TongJiListData)
|
|
|
+ {
|
|
|
+ var sjurl = item.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
|
|
|
+ sjurl = sjurl.Replace("－", "-");
|
|
|
+ GetEventHtml(sjurl, eventId, eventName, doc.DocumentNode.InnerText.Contains("球会友谊"));
|
|
|
+ }
|
|
|
+
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ });
|
|
|
+ //lock (g)
|
|
|
+ //{
|
|
|
+ // num++;
|
|
|
+ // Monitor.Pulse(g); //完成，通知等待队列,告知已完，执行下一个。
|
|
|
+ //}
|
|
|
});
|
|
|
- lock (g)
|
|
|
+ //lock (g)
|
|
|
+ //{
|
|
|
+ // while (num < max)
|
|
|
+ // {
|
|
|
+ // Monitor.Wait(g);//等待
|
|
|
+ // }
|
|
|
+ //}
|
|
|
+ //Trace.WriteLine("获取球队 完结");
|
|
|
+ while (true)
|
|
|
{
|
|
|
- while (num < max)
|
|
|
- {
|
|
|
- Monitor.Wait(g);//等待
|
|
|
- }
|
|
|
+ if (CommonHelper.ThreadsFinsh())
|
|
|
+ break;
|
|
|
}
|
|
|
- Trace.WriteLine("获取球队 完结");
|
|
|
try
|
|
|
{
|
|
|
if (teamList != null)
|
|
|
@@ -125,95 +135,92 @@ namespace FCS.Crawler.ZCLotteryTeam
|
|
|
return false;
|
|
|
}
|
|
|
var mainUrl = url;
|
|
|
- Task.Run(() =>
|
|
|
- {
|
|
|
- HtmlDocument doc = CommonHelper.GetHtml(mainUrl, new Dictionary<string, string>(), "足彩", "", 10000, 100);
|
|
|
|
|
|
- //获取左边的列表
|
|
|
- var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']");
|
|
|
- if (TongJiListData != null)
|
|
|
+ HtmlDocument doc = CommonHelper.GetHtml(mainUrl, new Dictionary<string, string>(), "足彩", "", 10000, 100);
|
|
|
+
|
|
|
+ //获取左边的列表
|
|
|
+ var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']");
|
|
|
+ if (TongJiListData != null)
|
|
|
+ {
|
|
|
+ foreach (var item in TongJiListData)
|
|
|
{
|
|
|
- foreach (var item in TongJiListData)
|
|
|
+ if (item.InnerText.Contains("球队列表") && item.InnerHtml != null)
|
|
|
{
|
|
|
- if (item.InnerText.Contains("球队列表") && item.InnerHtml != null)
|
|
|
+ //获取当前赛事的球队列表
|
|
|
+ HtmlDocument teamHtml = new HtmlDocument();
|
|
|
+ teamHtml.LoadHtml(item.InnerHtml);
|
|
|
+ var teams = teamHtml.DocumentNode.SelectNodes("//ul/a");
|
|
|
+ if (teams != null)
|
|
|
{
|
|
|
- //获取当前赛事的球队列表
|
|
|
- HtmlDocument teamHtml = new HtmlDocument();
|
|
|
- teamHtml.LoadHtml(item.InnerHtml);
|
|
|
- var teams = teamHtml.DocumentNode.SelectNodes("//ul/a");
|
|
|
- if (teams != null)
|
|
|
+
|
|
|
+ //球会友谊这个赛事里面的球队太多，单独提取，先读取出数据，后面插入到数据库再进行解析球队
|
|
|
+ if (IsQHYY)
|
|
|
{
|
|
|
- Task.Run(() =>
|
|
|
+ foreach (var teamitem in teams)
|
|
|
{
|
|
|
- //球会友谊这个赛事里面的球队太多，单独提取，先读取出数据，后面插入到数据库再进行解析球队
|
|
|
- if (IsQHYY)
|
|
|
+ int index = teams.IndexOf(item);
|
|
|
+ if (teamitem.InnerHtml != null && teamitem.InnerHtml != "")
|
|
|
{
|
|
|
- foreach (var teamitem in teams)
|
|
|
- {
|
|
|
- int index = teams.IndexOf(item);
|
|
|
- if (teamitem.InnerHtml != null && teamitem.InnerHtml != "")
|
|
|
- {
|
|
|
- var teamhref = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
|
|
|
- var teamName = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")).Value;
|
|
|
+ var teamhref = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
|
|
|
+ var teamName = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")).Value;
|
|
|
+
|
|
|
+ teamhref = teamhref.Replace("－", "-");
|
|
|
|
|
|
- teamhref = teamhref.Replace("－", "-");
|
|
|
+ var listcont = qhyyteamList.ToList().Where(o => o.Name.Trim() == teamName.Trim()).Count();
|
|
|
+ if (listcont == 0)
|
|
|
+ {
|
|
|
+ F_Team f_Team1 = new F_Team();
|
|
|
+ f_Team1.Id = Guid.NewGuid().ToString();
|
|
|
+ f_Team1.Name = teamName;
|
|
|
+ f_Team1.EventId = eventId;
|
|
|
+ f_Team1.Remark = teamhref;
|
|
|
+ qhyyteamList.Add(f_Team1);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ else
|
|
|
+ {
|
|
|
+ foreach (var teamitem in teams)
|
|
|
+ {
|
|
|
+ int index = teams.IndexOf(item);
|
|
|
+ if (teamitem.InnerHtml != null && teamitem.InnerHtml != "")
|
|
|
+ {
|
|
|
+ var teamhref = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
|
|
|
+ var teamName = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")).Value;
|
|
|
|
|
|
- var listcont = qhyyteamList.ToList().Where(o => o.Name.Trim() == teamName.Trim()).Count();
|
|
|
+ teamhref = teamhref.Replace("－", "-");
|
|
|
+ lock (teamList)
|
|
|
+ {
|
|
|
+ try
|
|
|
+ {
|
|
|
+ var listcont = teamList.ToList().Where(o => o.Name.Trim() == teamName.Trim()).Count();
|
|
|
if (listcont == 0)
|
|
|
{
|
|
|
F_Team f_Team1 = new F_Team();
|
|
|
f_Team1.Id = Guid.NewGuid().ToString();
|
|
|
f_Team1.Name = teamName;
|
|
|
f_Team1.EventId = eventId;
|
|
|
- f_Team1.Remark = teamhref;
|
|
|
- qhyyteamList.Add(f_Team1);
|
|
|
+ f_Team1.Remark = url;
|
|
|
+ teamList.Add(f_Team1);
|
|
|
}
|
|
|
}
|
|
|
- }
|
|
|
- }
|
|
|
- else
|
|
|
- {
|
|
|
- foreach (var teamitem in teams)
|
|
|
- {
|
|
|
- int index = teams.IndexOf(item);
|
|
|
- if (teamitem.InnerHtml != null && teamitem.InnerHtml != "")
|
|
|
- {
|
|
|
- var teamhref = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
|
|
|
- var teamName = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")).Value;
|
|
|
-
|
|
|
- teamhref = teamhref.Replace("－", "-");
|
|
|
- lock (teamList)
|
|
|
- {
|
|
|
- try
|
|
|
- {
|
|
|
- var listcont = teamList.ToList().Where(o => o.Name.Trim() == teamName.Trim()).Count();
|
|
|
- if (listcont == 0)
|
|
|
- {
|
|
|
- F_Team f_Team1 = new F_Team();
|
|
|
- f_Team1.Id = Guid.NewGuid().ToString();
|
|
|
- f_Team1.Name = teamName;
|
|
|
- f_Team1.EventId = eventId;
|
|
|
- f_Team1.Remark = url;
|
|
|
- teamList.Add(f_Team1);
|
|
|
- }
|
|
|
- }
|
|
|
- catch (Exception ex) { continue; }
|
|
|
+ catch (Exception ex) { continue; }
|
|
|
|
|
|
|
|
|
- }
|
|
|
- if (teamhref != "")
|
|
|
- {
|
|
|
- GetTeamHtml(teamhref, teamName, eventId);
|
|
|
- }
|
|
|
- }
|
|
|
+ }
|
|
|
+ if (teamhref != "")
|
|
|
+ {
|
|
|
+ GetTeamHtml(teamhref, teamName, eventId);
|
|
|
}
|
|
|
}
|
|
|
- });
|
|
|
+ }
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
- });
|
|
|
+ }
|
|
|
+
|
|
|
return true;
|
|
|
}
|
|
|
|
lyl