using System; using System.Collections.Generic; using System.Configuration; using System.Diagnostics; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using FCS.Common; using FCS.Interface; using FCS.Models; using HtmlAgilityPack; using Quartz; namespace FCS.Crawler.PlayersGameData { class PlayersGameDataJob : CommonJob, IJob { public PlayersGameDataJob() { log = new LogHelper(); services = IOC.Resolve(); } public void Execute(IJobExecutionContext context) { //Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); // Click(); } /// /// 跟新服务 /// public void Click() { playerslist = services.Query().ToList(); teamlist = services.Query().ToList(); eventlist = services.Query().ToList(); int num = 0; int max = playerslist.Count; playerslist.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("获取球员数据 开始:" + playerslist.IndexOf(p)); Getbjop(p.Remark, p.Id); }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } num = 0; max = GameData.Count; GameData.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("更换Id 开始:" + GameData.IndexOf(p)); F_PlayersGameData pd = new F_PlayersGameData(); pd = p; var events = eventlist.Where(a => a.Name.Trim() == p.EventName).FirstOrDefault(); pd.EventId = events == null ? "" : events.Id; var team = teamlist.Where(a => a.Name.Trim() == p.HomeTeamName).FirstOrDefault(); pd.HomeTeamId = team == null ? "" : team.Id; team = teamlist.Where(a => a.Name.Trim() == p.VisitingTeamName).FirstOrDefault(); pd.VisitingTeamId = team == null ? "" : team.Id; lock (GameData_New) { GameData_New.Add(p); } }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } var sss = Convert.ToDateTime(DateTime.Now.ToShortDateString()); //只插入当前天数的球员比赛数据 GameData_New = GameData_New.Where(a => a.RaceTime > Convert.ToDateTime(DateTime.Now.AddDays(-30).ToShortDateString())).ToList(); Trace.WriteLine("插入球员记录数据 开始:" + GameData_New.Count); if (GameData_New.Count > 0) services.SqlBulkCopyAdd(GameData_New); } F_PlayersGameData g = new F_PlayersGameData(); public void DoMainUrl() { playerslist = services.Query().Take(70000).ToList(); teamlist = services.Query().ToList(); eventlist= services.Query().ToList(); int num = 0; int max = playerslist.Count; playerslist.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("获取球员数据 开始:" + playerslist.IndexOf(p)); Getbjop(p.Remark,p.Id); }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } num = 0; max = GameData.Count; GameData.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("更换Id 开始:" + GameData.IndexOf(p)); F_PlayersGameData pd = new F_PlayersGameData(); pd = p; var events = eventlist.Where(a => a.Name.Trim() == p.EventName).FirstOrDefault(); pd.EventId = events==null?"": events.Id; var team = teamlist.Where(a => a.Name.Trim() == p.HomeTeamName).FirstOrDefault(); pd.HomeTeamId = team == null ? "" : team.Id; team = teamlist.Where(a => a.Name.Trim() == p.VisitingTeamName).FirstOrDefault(); pd.VisitingTeamId = team == null ? "" : team.Id; lock (GameData_New) { GameData_New.Add(p); } }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } Trace.WriteLine("插入球员记录数据 开始:" + GameData_New.Count); if (GameData_New.Count > 0) services.SqlBulkCopyAdd(GameData_New); DoMainUrl1(); } public void DoMainUrl1() { GameData_New = new List(); playerslist = services.Query().Skip(70000).ToList(); int num = 0; int max = playerslist.Count; playerslist.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("获取球员数据 开始:" + playerslist.IndexOf(p)); Getbjop(p.Remark, p.Id); }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } num = 0; max = GameData.Count; GameData.ForEach(async p => { await Task.Run(() => { Trace.WriteLine("更换Id 开始:" + GameData.IndexOf(p)); F_PlayersGameData pd = new F_PlayersGameData(); pd = p; var events = eventlist.Where(a => a.Name.Trim() == p.EventName).FirstOrDefault(); pd.EventId = events == null ? "" : events.Id; var team = teamlist.Where(a => a.Name.Trim() == p.HomeTeamName).FirstOrDefault(); pd.HomeTeamId = team == null ? "" : team.Id; team = teamlist.Where(a => a.Name.Trim() == p.VisitingTeamName).FirstOrDefault(); pd.VisitingTeamId = team == null ? "" : team.Id; lock (GameData_New) { GameData_New.Add(p); } }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num != max) { Monitor.Wait(g);//等待 } } Trace.WriteLine("插入球员记录数据 开始:" + GameData_New.Count); if (GameData_New.Count > 0) services.SqlBulkCopyAdd(GameData_New); } public static List GameData = new List(); public static List GameData_New = new List(); public static List playerslist = new List(); public static List teamlist = new List(); public static List eventlist = new List(); public void Getbjop(string url,string Id) { url = url.Replace(":80", ""); var htmlResource = CommonHelper.GetHtmlString(url, "足彩"); HtmlDocument doc = new HtmlDocument(); if (htmlResource == null) return; doc.LoadHtml(htmlResource); //球员集合 HtmlNodeCollection sjHtml = doc.DocumentNode.SelectNodes("//*[@class='zstab']/tr"); if (sjHtml == null) return; for (int i = 0; i < sjHtml.Count; i++) { if (i == 0 || i == sjHtml.Count - 1) continue; string p = sjHtml[i].InnerHtml; HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(p); try { HtmlNodeCollection Htmltd = doc.DocumentNode.SelectNodes("//td"); if (Htmltd == null) return; F_PlayersGameData data = new F_PlayersGameData(); data.Id = CommonHelper.GetGuid(); data.EventName = Htmltd[0].InnerText.Trim(); data.EventId = Htmltd[0].InnerText.Trim(); data.RaceTime = Convert.ToDateTime(Htmltd[1].InnerText.Trim()); data.HomeTeamName = Htmltd[2].InnerText.Trim(); data.VisitingTeamName = Htmltd[4].InnerText.Trim(); data.HomeTeamId = Htmltd[2].InnerText.Trim(); data.VisitingTeamId = Htmltd[4].InnerText.Trim(); data.Score = Htmltd[3].InnerText.Trim(); data.Goal = Convert.ToInt32(Htmltd[5].InnerText.Trim()); data.PenaltyKick = Convert.ToInt32(Htmltd[6].InnerText.Trim()); data.Oolong = Convert.ToInt32(Htmltd[7].InnerText.Trim()); data.YellowCard = Convert.ToInt32(Htmltd[8].InnerText.Trim()); data.RedCard = Convert.ToInt32(Htmltd[9].InnerText.Trim()); data.Remark = url; data.PlayersId = Id; data.CreateDateTime = DateTime.Now; Insert(data); } catch (Exception e) { continue; } } } /// /// 插入数据 /// /// /// /// public void Insert(F_PlayersGameData data) { lock (GameData) { GameData.Add(data); } } } }