using System; using System.Collections.Generic; using FCS.Common; using FCS.Interface; using FCS.Models; using HtmlAgilityPack; using Quartz; using System.Data; using System.Linq; using System.Threading.Tasks; using System.Threading; using System.Diagnostics; namespace FCS.Crawler.ZCMatchRankingList { /// /// 进球数单双统计 /// public class GoalSingleDbListJob : CommonJob, IJob { private static List SingleDoubleLists = new List(); private static List TeamList = new List(); public GoalSingleDbListJob() { log = new LogHelper(); services = IOC.Resolve(); } public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); GetAll(); } #region 更新数据 /// /// 获取所有数据 /// public void GetAll() { GetSqlString(GetAllEventUrl,""); } /// /// 更新数据,默认为今年数据 /// /// public void Click(string Year = "") { if (Year == "") { Year = DateTime.Now.Year.ToString(); } var sql = string.Format(GetNewEventUrl, Year); GetSqlString(sql, Year); } #endregion /// /// 解析数据 /// /// private void GetSqlString(string sqlstring,string Year) { ThreadPool.SetMinThreads(10, 10); ThreadPool.SetMaxThreads(500, 500); TeamList = services.GetTeamList(); var ds = SqlHelper.ExecuteDataset(CommandType.Text, sqlstring); if (ds != null && ds.Tables.Count > 0) { for (int i = 0; i < ds.Tables[0].Rows.Count; i++) { //拿去赛事组的链接 var url = ds.Tables[0].Rows[i]["Remark"].ToString(); var eventId = ds.Tables[0].Rows[i]["EventId"].ToString(); var season = ds.Tables[0].Rows[i]["Season"].ToString(); ////获取到总进球单双的链接 Task.Run(() => { HtmlDocument doc = CommonHelper.GetHtml(url, new Dictionary(), "", "", 90000, 10); var TJList = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']"); if (TJList != null && TJList.Count() != 0 && TJList[0].InnerText.Contains("赛事统计")) { HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(TJList[0].InnerHtml); var liList = doc1.DocumentNode.SelectNodes("//ul/li/a").ToList(); var Glod = liList.Where(o => o.InnerText.Contains("进球单双")).First(); if (Glod != null) { var jqdsurl = Glod.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value.ToString(); Task.Run(() => { GetJQDSHtml(jqdsurl, eventId, season); }); } } }); } int maxWorkerThreads, workerThreads; int maxportThreads, portThreads; while (true) { /* GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。 */ ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads); ThreadPool.GetAvailableThreads(out workerThreads, out portThreads); Thread.Sleep(1000); Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads)); if (maxWorkerThreads - workerThreads == 0) { Console.WriteLine("Thread Finished!"); break; } } try { if (Year!="") { services.Delete(" and Season like '"+Year+"%'"); } services.SqlBulkCopyAdd(SingleDoubleLists); Trace.WriteLine("F_SingleDoubleList更新完毕"); } catch (Exception ex) { throw; } } } /// /// 解析进球单双页面 /// /// /// /// public void GetJQDSHtml(string url, string eventId, string season) { if (url.Contains("wwaattssuunn")) { return; } HtmlDocument doc = CommonHelper.GetHtml(url,new Dictionary(),"","",9000,10); var trData = doc.DocumentNode.SelectNodes("//*[@class='table_out']/table/tbody/tr"); if (trData == null || trData.Count() == 0) return; //读取进球单双信息并加入到数据库 foreach (var tritem in trData) { try { HtmlDocument doc11 = new HtmlDocument(); doc11.LoadHtml(tritem.InnerHtml); var tdList = doc11.DocumentNode.SelectNodes("//td"); if (tdList == null || tdList.Count == 0) continue; F_SingleDoubleList f_SingleDoubleList = new F_SingleDoubleList(); f_SingleDoubleList.Id = Guid.NewGuid().ToString(); f_SingleDoubleList.EventId = eventId; f_SingleDoubleList.Season = season; f_SingleDoubleList.Sort = tdList[0].InnerText.Trim() == null ? 0 : int.Parse(tdList[0].InnerText.Trim().ToString()); var TeamName = tdList[1].InnerText.Trim().ToString(); var Team = TeamList.Where(o => o.Name == TeamName); f_SingleDoubleList.TeamId = Team.Count() > 0 ? Team.FirstOrDefault().Id : ""; if (Team.Count() == 0) { f_SingleDoubleList.Remark = TeamName; } f_SingleDoubleList.OneNum = tdList.Count()>3? int.Parse(tdList[2].InnerText.Trim().ToString()):0; f_SingleDoubleList.ZeroNum = tdList.Count() > 4 ? int.Parse(tdList[3].InnerText.Trim().ToString()) : 0; f_SingleDoubleList.TwoNum = tdList.Count() > 5 ? int.Parse(tdList[4].InnerText.Trim().ToString()) : 0; f_SingleDoubleList.ThreeNum = tdList.Count() > 6 ? int.Parse(tdList[5].InnerText.Trim().ToString()) : 0; f_SingleDoubleList.FourNum = tdList.Count() > 7 ? int.Parse(tdList[6].InnerText.Trim().ToString()) : 0; f_SingleDoubleList.OddNum = tdList.Count() > 8 ? int.Parse(tdList[7].InnerText.Trim().ToString()) : 0; f_SingleDoubleList.EvenNum = tdList.Count() > 9 ? int.Parse(tdList[8].InnerText.Trim().ToString()) : 0; SingleDoubleLists.Add(f_SingleDoubleList); } catch (Exception ex) { throw; } } } #region SQL语句 /// /// 获取所有的赛事 /// private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null"; /// /// 获取最新两年的数据赛事 /// private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null and Season like '%{0}%'"; private static string DeleteNewEventUrl = @"delete F_SingleDoubleList where Season like '{0}%'"; #endregion } }