using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net; using System.Text; using System.Threading; using System.Threading.Tasks; using FCS.Common; using FCS.Interface; using FCS.Models; using HtmlAgilityPack; using Quartz; namespace FCS.Crawler.ZCLotteryGames { class GamesJob : CommonJob, IJob { public GamesJob() { log = new LogHelper(); services = IOC.Resolve(); g = new F_Grouping(); } public void Execute(IJobExecutionContext context) { //Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); // Click(); } public static List gamelist = new List(); public static List urllist = new List(); public static List urllist2 = new List(); public static List Group = new List(); public static List Group1 = new List(); public static List GroupParent = new List(); public static List teamlist = new List(); public static List GroupList = null; public static List GroupList1 = null; public static List Gamelist = null; public static List gamesList = new List(); public static List gamesList_new = new List(); public static List gamesList_new1 = new List(); public static List gamesList_new2 = new List(); public static List listint = new List();//需要删除的比赛PK F_Grouping g; public List Click() { string Season = DateTime.Now.ToString("yyyy"); Group = services.Query().ToList(); Group1 = services.Query(" and Id in( select aa.Id from F_Grouping as aa left join F_Games as b on aa.Id=b.GroupingId where b.[Status]!='a058ec2ec96ae811ab3100ac6a4a134f' and aa.Season like '%{0}%' and b.Season like '%{0}%' group by aa.Id)".FormatMe(Season)).ToList(); Group1.AddRange(services.Query(" and Id in ( select aa.Id from F_Grouping as aa left join F_Games as b on aa.Id=b.GroupingId where b.Id is null and aa.Season like '%{0}%' group by aa.Id)".FormatMe(Season)).ToList()); Gamelist = services.Query(" AND Season LIKE '%{0}%'".FormatMe(Season)).ToList(); teamlist = services.Query().ToList(); GroupList = Group1.Where(a => !a.Remark.Contains("/cup/41/")&&((a.ParentId.Length == 0 && a.Remark.Contains("/cup/")) || a.Remark.Contains("/league/"))).ToList(); GroupParent = services.Query("and ParentId!=''").ToList(); // GetClick("http://saishi.zgzcw.com:80/soccer/cup/1183/2018/", "半准决赛", "55a70f4a1dcbe811894c2c4d54d3f5a9"); int max = GroupList.Count; int num = 0; //比赛 GroupList.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("获取比赛 开始：" + GroupList.IndexOf(p)); GetClick(p.Remark, p.Name.Trim(), p.Id); }); lock (g) { num++; Monitor.Pulse(g); //完成，通知等待队列,告知已完，执行下一个。 } }); lock (g) { while (num < max) { Monitor.Wait(g);//等待 } } Trace.WriteLine("获取比赛 完结"); //球队联谊 //var q = Group.Where(a => a.Remark.Contains("/cup/41/") && a.Season.Contains(Season)).ToList(); //List ulist = new List(); //foreach (var item in q) //{ // ulist.Add(item.Remark); //} //ulist = ulist.Distinct().ToList(); //foreach (var item in ulist) //{ // GetCupData_yy(item, q); //} ////检查缺漏 List new_urllist = new List(); new_urllist = urllist; max = new_urllist.Count; num = 0; urllist = new List(); new_urllist.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("获取比赛 缺漏 开始：" + new_urllist.IndexOf(p)); var query = p.Split('|'); GetLeagueData(query[0], query[1], query[2]); }); lock (g) { num++; Monitor.Pulse(g); //完成，通知等待队列,告知已完，执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } // Gamelist= Gamelist.Where((x, i) => Gamelist.FindIndex(z => z.GroupingId == x.GroupingId && z.VisitingTeamId == x.VisitingTeamId && z.HomeTeamId == x.HomeTeamId && z.HomeTeamScore_All == x.HomeTeamScore_All && z.VisitingTeamScore_All == x.VisitingTeamScore_All && z.HomeTeamScore_Half == x.HomeTeamScore_Half && z.VisitingTeamScore_Half == x.VisitingTeamScore_Half && z.EventId == x.EventId && z.Season == x.Season) == i).ToList();//Lambda表达式去重 gamesList =gamesList.Where((x, i) => gamesList.FindIndex(z => z.GroupingId == x.GroupingId && z.VisitingTeamId == x.VisitingTeamId && z.HomeTeamId == x.HomeTeamId && z.HomeTeamScore_All == x.HomeTeamScore_All && z.VisitingTeamScore_All == x.VisitingTeamScore_All && z.HomeTeamScore_Half == x.HomeTeamScore_Half && z.VisitingTeamScore_Half == x.VisitingTeamScore_Half && z.EventId == x.EventId && z.Season == x.Season) == i).ToList();//Lambda表达式去重 //{ // max = gamesList.Count; // num = 0; // int sss = 0; // //比较比赛 // gamesList.ForEach(async p => // { // await Task.Run(() => // { // var istrue = Gamelist.Where(a => a.Season == p.Season && a.GroupingId == p.GroupingId && a.HomeTeamId == p.HomeTeamId && a.VisitingTeamId == p.VisitingTeamId && a.EventId == p.EventId && a.StartDateTime == p.StartDateTime).ToList(); // //if(Gamelist.Where(a=>a.HomeTeamId== "ff8d81fa-3444-4e5d-955c-79049a722daa"&&a.VisitingTeamId== "5629c466-a3e4-4a1a-9826-0a5c86244f08"&&a.CreateDateTime>)) // if (istrue.Count > 0) // {//已有比赛 判断是否需要更新数据 // var onegame = istrue.FirstOrDefault(); // if (onegame.HomeTeamScore_All != p.HomeTeamScore_All || onegame.HomeTeamScore_Half != p.HomeTeamScore_Half || onegame.VisitingTeamScore_All != p.VisitingTeamScore_All || onegame.VisitingTeamScore_Half != p.VisitingTeamScore_Half || onegame.Status != "a058ec2ec96ae811ab3100ac6a4a134f") // { // lock (gamesList_new) // { // p.Id = onegame.Id; // gamesList_new.Add(p); // gamesList_new2.Add(p); // } // foreach (var item in istrue) // { // lock (listint) // { // listint.Add(item.PK); // } // } // } // else // { // //数据不做修改 // } // } // else // {//没有次比赛 新增 // lock (gamesList_new) // { // gamesList_new.Add(p); // } // } // }); // lock (locker) // { // num++; // Monitor.Pulse(locker); //完成，通知等待队列,告知已完，执行下一个。 // } // }); // lock (locker) // { // while (num < max) // { // Monitor.Wait(locker);//等待 // } // } //} var group = gamesList.GroupBy(a => a.GroupingId).Select(g => g.Last()).ToList(); foreach (var item in group) { var new_group_game=gamesList.Where(a => a.GroupingId == item.GroupingId).ToList(); var group_game = Gamelist.Where(a => a.GroupingId == item.GroupingId).ToList(); foreach (var item1 in new_group_game) { switch (group_game.Where(a => a.GroupingId == item.GroupingId && a.HomeTeamId == item1.HomeTeamId && a.VisitingTeamId == item1.VisitingTeamId && a.StartDateTime == item1.StartDateTime && a.Status == item1.Status).ToList().Count) { case 0: gamesList_new.Add(item1); break; case 1: var gamedetail = group_game.Where(a => a.GroupingId == item.GroupingId && a.HomeTeamId == item1.HomeTeamId && a.VisitingTeamId == item1.VisitingTeamId && a.StartDateTime == item1.StartDateTime && a.Status == item1.Status).FirstOrDefault(); var new_game = item1; new_game.Id = gamedetail.Id; gamesList_new.Add(new_game); listint.Add(gamedetail.PK); break; default: var gamedetail1 = group_game.Where(a => a.GroupingId == item.GroupingId && a.HomeTeamId == item1.HomeTeamId && a.VisitingTeamId == item1.VisitingTeamId && a.StartDateTime == item1.StartDateTime && a.Status == item1.Status).ToList(); var new_game1 = item1; new_game1.Id = gamedetail1.OrderByDescending(a=>a.CreateDateTime).FirstOrDefault().Id; gamesList_new.Add(new_game1); foreach (var item2 in gamedetail1) { listint.Add(item2.PK); } break; } } foreach (var item1 in group_game) { listint.Add(item1.PK); } } Thread.Sleep(5000); var gamesList_new1 = gamesList_new.Where(a => a.StartDateTime < DateTime.Now.AddDays(5)).ToList(); var salist1 = gamesList_new.Where((x, i) => gamesList_new.FindIndex(z => z.GroupingId == x.GroupingId&&z.VisitingTeamId==x.VisitingTeamId && z.HomeTeamId == x.HomeTeamId && z.HomeTeamScore_All == x.HomeTeamScore_All && z.VisitingTeamScore_All == x.VisitingTeamScore_All && z.HomeTeamScore_Half == x.HomeTeamScore_Half && z.VisitingTeamScore_Half == x.VisitingTeamScore_Half&&z.EventId==x.EventId&&z.Season==x.Season && z.Id == x.Id) == i).ToList();//Lambda表达式去重 //gamesList2 = gamesList_new2.Where(a => a.StartDateTime < Convert.ToDateTime(DateTime.Now.ToString("yyyy-MM-dd"))&& a.StartDateTime > Convert.ToDateTime(DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd"))).ToList(); Trace.WriteLine("插入数据 开始：" + salist1.Count); services.SqlBulkCopyAdd(salist1); Trace.WriteLine("插入数据 结束：" + salist1.Count); Trace.WriteLine("删除数据 开始：" + listint.Count); if (listint.Count > 0) { int pg = listint.Count % 10000 == 0 ? listint.Count / 10000 : listint.Count / 10000 + 1; for (int i = 0; i < pg; i++) { var listint1 = listint.Skip(i * 10000).Take(10000).ToList(); services.Delete(" and PK in (" + string.Join(",", listint1) + ")"); } } Trace.WriteLine("删除数据 结束：" + listint.Count); List rlist = new List(); var querygame = services.Query(" and Id in (select Id from ( select Id,COUNT(Id) as num from F_Games group by Id) as s where s.num>1 ) ").ToList(); var q1 = querygame.Where((x, i) => querygame.FindIndex(z => z.Id == x.Id) == i).ToList();//Lambda表达式去重 foreach (var item in querygame) { if (q1.Where(a => a.PK == item.PK).ToList().Count > 0) { continue; } else { rlist.Add(item.PK); } } if (rlist.Count > 0) { int pg = rlist.Count % 10000 == 0 ? rlist.Count / 10000 : rlist.Count / 10000 + 1; for (int i = 0; i < pg; i++) { var listint1 = rlist.Skip(i * 10000).Take(10000).ToList(); services.Delete(" and PK in (" + string.Join(",", listint1) + ")"); } } return gamesList_new1; } public void DoMainUrl() { Group = services.Query().ToList(); teamlist = services.Query().ToList(); ThreadPool.SetMinThreads(1, 1); ThreadPool.SetMaxThreads(100, 100); GroupList = Group.Where(a => a.Remark.Contains("cup") && !a.Remark.Contains("/cup/41/")).ToList(); //比赛杯 GroupList.ForEach(p => { Task.Run(() => { Trace.WriteLine("获取比赛杯 开始：" + GroupList.IndexOf(p)); GetCupData(p.Remark, p.Name.Trim(), p.Id); }); }); while (true) { if (CommonHelper.ThreadsFinsh()) break; } Trace.WriteLine("获取比赛杯 完结"); //球队联谊 var q = Group.Where(a => a.Remark.Contains("cup") && a.Remark.Contains("/cup/41/")).ToList(); List ulist = new List(); foreach (var item in q) { ulist.Add(item.Remark); } ulist = ulist.Distinct().ToList(); foreach (var item in ulist) { GetCupData_yy(item, q); } //国家比赛 GroupList1 = Group.Where(a => a.Remark.Contains("league") && a.Name != "常规赛").ToList(); GroupList1.ForEach(p => { Task.Run(() => { Trace.WriteLine("获取比赛 开始：" + GroupList1.IndexOf(p)); GetLeagueData(p.Remark, p.Name.Trim(), p.Id); }); }); while (true) { if (CommonHelper.ThreadsFinsh()) break; } //检查缺漏 List new_urllist = new List(); new_urllist = urllist; urllist = new List(); foreach (var item in new_urllist) { var query = item.Split('|'); GetLeagueData(query[0], query[1], query[2]); } Trace.WriteLine("插入数据 开始：" + gamesList.Count); services.SqlBulkCopyAdd(gamesList); Trace.WriteLine("插入数据 结束：" + gamesList.Count); } /// /// 某某杯 /// /// public void GetCupData(string url, string currentRound, string Id) { try { Trace.WriteLine("某某杯 开始：" + url); string yurl = url; url = url.Replace(":80", ""); var htmlResource = CommonHelper.GetHtmlString(url, "足彩"); HtmlDocument doc = new HtmlDocument(); if (htmlResource == null) return; doc.LoadHtml(htmlResource); //比赛分组集合 HtmlNodeCollection GamegroupHtml = doc.DocumentNode.SelectNodes("//*[@id='tabs9']/li"); foreach (var item in GamegroupHtml.Where(a => a.InnerText.Trim() == currentRound)) {//获取父下所有子分组+比赛 string shuju1 = ""; if (GroupParent.Where(a => a.ParentId == Id).ToList().Count > 0) {//有子集比赛集合 HtmlNodeCollection groupHtml = doc.DocumentNode.SelectNodes("//*[@id='tabs9_" + item.Id + "']"); if (groupHtml == null) return; HtmlDocument doc_1 = new HtmlDocument(); doc_1.LoadHtml(groupHtml[0].OuterHtml); HtmlNodeCollection groupHtmlnew = doc_1.DocumentNode.SelectNodes("//li"); foreach (var item1 in groupHtmlnew) { var zj = GroupParent.Where(a => a.ParentId == Id && a.Name == item1.InnerText.Trim()).ToList(); if(zj.Count==0) continue; //获取所有子集 HtmlNodeCollection itemlist_1 = doc_1.DocumentNode.SelectNodes("//*[@id='div_" + item1.Id + "']/table"); if (itemlist_1 == null) return; HtmlDocument doc2 = new HtmlDocument(); doc2.LoadHtml(itemlist_1[0].OuterHtml); HtmlNodeCollection itemlist = doc2.DocumentNode.SelectNodes("//tr"); foreach (HtmlNode node in itemlist) { var list = node.SelectNodes("td"); if (list == null) continue; string shuju = shuju1; foreach (var item3 in list) { shuju += item3.InnerText.Trim() + "|"; } //gamelist.Add(shuju); var listurl = url.Split('/'); string season = listurl[6]; HtmlDocument doc3 = new HtmlDocument(); doc3.LoadHtml(list.Last().InnerHtml); HtmlNodeCollection itemlist2 = doc3.DocumentNode.SelectNodes("//a"); Insert(shuju, zj[0].Id, season, itemlist2[0].Attributes["href"].Value); } } } else { //比赛集合 HtmlNodeCollection GameHtml = doc.DocumentNode.SelectNodes("//*[@id='tabs9_" + item.Id + "']"); foreach (var item1 in GameHtml) { HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(item1.OuterHtml); HtmlNodeCollection itemlist = doc1.DocumentNode.SelectNodes("//table").Last().SelectNodes("//tr"); foreach (HtmlNode node in itemlist) { var list = node.SelectNodes("td"); if (list == null) continue; string shuju = shuju1; foreach (var item3 in list) { shuju += item3.InnerText.Trim() + "|"; } //gamelist.Add(shuju); var listurl = url.Split('/'); string season = listurl[6]; HtmlDocument doc2 = new HtmlDocument(); doc2.LoadHtml(list.Last().InnerHtml); HtmlNodeCollection itemlist2 = doc2.DocumentNode.SelectNodes("//a"); Insert(shuju, Id, season, itemlist2[0].Attributes["href"].Value); } } } } } catch (Exception e) { Trace.WriteLine("某某杯 发生异常：" + url); } } public void GetClick(string url, string currentRound, string Id) { if (url.Contains("cup")) { GetCupData(url, currentRound, Id); } else { GetLeagueData(url, currentRound, Id); } } /// /// 某某杯 友谊 /// /// public void GetCupData_yy(string url, List glist) { string currentRound, Id; try { Trace.WriteLine("某某杯 开始：" + url); url = url.Replace(":80", ""); var htmlResource = CommonHelper.GetHtmlString(url, "足彩"); HtmlDocument doc = new HtmlDocument(); if (htmlResource == null) return; doc.LoadHtml(htmlResource); //比赛分组集合 HtmlNodeCollection GamegroupHtml = doc.DocumentNode.SelectNodes("//*[@id='tabs9']/li"); foreach (var item in GamegroupHtml) { string shuju1 = ""; //比赛集合 HtmlNodeCollection GameHtml = doc.DocumentNode.SelectNodes("//*[@id='tabs9_" + item.Id + "']"); foreach (var item1 in GameHtml) { HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(item1.OuterHtml); HtmlNodeCollection itemlist = doc1.DocumentNode.SelectNodes("//table").Last().SelectNodes("//tr"); foreach (HtmlNode node in itemlist) { var list = node.SelectNodes("td"); if (list == null) continue; string shuju = shuju1; foreach (var item3 in list) { shuju += item3.InnerText.Trim() + "|"; } //gamelist.Add(shuju); var listurl = url.Split('/'); string season = listurl[6]; HtmlDocument doc2 = new HtmlDocument(); doc2.LoadHtml(list.Last().InnerHtml); HtmlNodeCollection itemlist2 = doc2.DocumentNode.SelectNodes("//a"); var sss = glist.Where(a => a.Name.Trim() == item.InnerText.Trim() && a.Remark.Replace(":80", "") == url).FirstOrDefault(); Id = glist.Where(a => a.Name.Trim() == item.InnerText.Trim() && a.Remark.Replace(":80", "") == url).FirstOrDefault() == null ? "" : glist.Where(a => a.Name.Trim() == item.InnerText.Trim() && a.Remark.Replace(":80", "") == url).FirstOrDefault().Id; Insert(shuju, Id, season, itemlist2[0].Attributes["href"].Value); } } } } catch (Exception e) { Trace.WriteLine("某某杯 发生异常：" + url); } } /// /// 国家赛事 /// /// public void GetLeagueData(string url, string currentRound, string Id) { string msg = ""; try { Trace.WriteLine("国家赛事 开始：" + url); url = url.Replace(":80", ""); int Round = -1; try { Round = Convert.ToInt32(currentRound); } catch (Exception e) { Round = -1; } if (Round != -1) { var listurl = url.Split('/'); string source_league_id = listurl[5]; string season = listurl[6]; var html = CommonHelper.GetPostHtmlString(url, new HttpItem { Url = "http://saishi.zgzcw.com/summary/liansaiAjax.action", Method = "Post", WebProxy = new WebProxy(CommonHelper.GetIp()), FormData = new Dictionary() { { "source_league_id" , source_league_id }, { "currentRound",currentRound}, { "season",season}, { "seasonType",""}, }, ContentType = "application/x-www-form-urlencoded", }); //常规赛 msg = html.Html; HtmlDocument doc = new HtmlDocument(); if (html.Html == null) return; doc.LoadHtml(html.Html); //比赛集合 HtmlNodeCollection itemlist = doc.DocumentNode.SelectNodes("//table/tr"); foreach (HtmlNode node in itemlist) { var list = node.SelectNodes("td"); string shuju = ""; foreach (var item3 in list) { shuju += item3.InnerText.Trim() + "|"; } //gamelist.Add(shuju); HtmlDocument doc2 = new HtmlDocument(); doc2.LoadHtml(list.Last().InnerHtml); HtmlNodeCollection itemlist2 = doc2.DocumentNode.SelectNodes("//a"); Insert(shuju, Id, season, itemlist2[0].Attributes["href"].Value); } } else { var listurl = url.Split('/'); string source_league_id = listurl[5]; string season = listurl[6]; var html = CommonHelper.GetPostHtmlString(url, new HttpItem { Url = "http://saishi.zgzcw.com/summary/liansaifjAction.action", Method = "Post", WebProxy = new WebProxy(CommonHelper.GetIp()), FormData = new Dictionary() { { "source_league_id" , source_league_id}, { "hostTeamId","0"}, { "guestTeamId","0"}, { "season",season}, { "seasonType","1"}, }, ContentType = "application/x-www-form-urlencoded", }); msg = html.Html; HtmlDocument doc = new HtmlDocument(); if (html.Html == null) return; doc.LoadHtml(html.Html); //附加比赛集合 HtmlNodeCollection itemlist = doc.DocumentNode.SelectNodes("//*[@id ='fjtab']/tr"); if (itemlist == null) { Trace.WriteLine("附加赛没有数据" + url); return; } foreach (HtmlNode node1 in itemlist) { var list1 = node1.SelectNodes("td"); string shuju = ""; foreach (var item3 in list1) { shuju += item3.InnerText.Trim() + "|"; } // gamelist.Add(shuju); HtmlDocument doc2 = new HtmlDocument(); doc2.LoadHtml(list1.Last().InnerHtml); HtmlNodeCollection itemlist2 = doc2.DocumentNode.SelectNodes("//a"); Insert(shuju, Id, season, itemlist2[0].Attributes["href"].Value); } } } catch (Exception e) { urllist.Add(url + "|" + currentRound + "|" + Id); Trace.WriteLine("某某杯 发生异常：" + url + "_:" + currentRound + ":_" + "__" + msg); } } /// /// 插入数据 /// /// /// /// public void Insert(string data, string Id, string Season, string url) { lock (gamesList) { var query = data.Split('|'); F_Games game = new F_Games(); game.Id = CommonHelper.GetGuid().ToString(); game.CreateDateTime = DateTime.Now; game.EventId = Group.Where(a => a.Id == Id).FirstOrDefault().EventId; game.EventType = Group.Where(a => a.Id == Id).FirstOrDefault().EventType; game.Heat = 0; var team = teamlist.Where(a => a.Name == query[1]).FirstOrDefault(); if (team == null) return; game.HomeTeamId = team == null ? "" : team.Id; game.HomeTeamScore_All = query[2].Split(':')[0].TryToInt32(); game.HomeTeamScore_Half = query[4].Split(':')[0].TryToInt32(); game.IsOvertime = false; game.Remark = url; game.Season = Season; game.StartDateTime = Convert.ToDateTime(query[0]); game.Status = query[4] == "-:-" ? g1 : query[2] == "-:-" ? g2 : g3; team = teamlist.Where(a => a.Name == query[3]).FirstOrDefault(); if (team == null) return; game.VisitingTeamId = team.Id; game.VisitingTeamScore_All = query[2].Split(':')[1].TryToInt32(); game.VisitingTeamScore_Half = query[4].Split(':')[1].TryToInt32(); game.GroupingId = Id; game.IsObtain = false; gamesList.Add(game); } } private string g1 = "d0da55f0c86ae811ab3100ac6a4a134f", g2 = "907dbd11c96ae811ab3100ac6a4a134f", g3 = "a058ec2ec96ae811ab3100ac6a4a134f"; } }