using System; using System.Collections.Generic; using FCS.Common; using FCS.Interface; using FCS.Models; using HtmlAgilityPack; using Quartz; using System.Data; using System.Linq; using System.Threading.Tasks; using System.Threading; using System.Diagnostics; namespace FCS.Crawler.ZCMatchRankingList { /// /// 射手榜排行 /// public class ShooterListJob : CommonJob, IJob { private static List ShooterLists = new List(); private static List TeamList = new List(); private static List PlayerList = new List(); private static List ContryList = new List(); private static List DateItem = new List(); public ShooterListJob() { log = new LogHelper(); services = IOC.Resolve(); } public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); GetAll(); } #region 更新数据 /// /// 获取所有数据 /// public void GetAll() { GetSqlString(GetAllEventUrl, ""); } /// /// 更新数据 /// /// public void Click(string Year = "") { if (Year == "") { Year = DateTime.Now.Year.ToString(); } var sql = string.Format(GetNewEventUrl, Year); GetSqlString(sql, Year); } #endregion /// /// 解析数据 /// /// private void GetSqlString(string sqlstring, string Year) { ThreadPool.SetMinThreads(10, 10); ThreadPool.SetMaxThreads(500, 500); TeamList = services.GetTeamList(); PlayerList = services.GetPlayerList(); ContryList = services.GetContryList(); var ds = SqlHelper.ExecuteDataset(CommandType.Text, sqlstring); if (ds != null && ds.Tables.Count > 0) { for (int i = 0; i < ds.Tables[0].Rows.Count; i++) { //拿去赛事组的链接 var url = ds.Tables[0].Rows[i]["Remark"].ToString(); var eventId = ds.Tables[0].Rows[i]["EventId"].ToString(); var season = ds.Tables[0].Rows[i]["Season"].ToString(); ////获取到总进球单双的链接 Task.Run(() => { HtmlDocument doc = CommonHelper.GetHtml(url, new Dictionary(), "", "", 10000, 100); var TJList = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']"); if (TJList != null && TJList.Count() != 0 && TJList[0].InnerText.Contains("赛事统计")) { HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(TJList[0].InnerHtml); var liList = doc1.DocumentNode.SelectNodes("//ul/li/a").ToList(); var Glod = liList.Where(o => o.InnerText.Contains("射手榜")).First(); if (Glod != null) { var shooturl = Glod.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value.ToString(); Task.Run(() => { GetList(shooturl, eventId, season); }); } } }); } int maxWorkerThreads, workerThreads; int maxportThreads, portThreads; while (true) { /* GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。 */ ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads); ThreadPool.GetAvailableThreads(out workerThreads, out portThreads); Thread.Sleep(1000); Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads)); if (maxWorkerThreads - workerThreads == 0) { Console.WriteLine("Thread Finished!"); break; } } try { if (Year != "") { services.Delete(" and Season like '%"+Year+"%'"); } services.SqlBulkCopyAdd(ShooterLists); Trace.WriteLine("F_ShooterList更新完毕"); } catch (Exception ex) { throw; } } } /// /// 解析射手榜信息 /// /// /// /// public void GetList(string url, string eventId, string season) { if (url.Contains("wwaattssuunn")) { return; } HtmlDocument doc = CommonHelper.GetHtml(url, new Dictionary(), "", "", 10000, 100); var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='zstab']"); if (TongJiListData == null || TongJiListData.Count() == 0) return; DateItem = services.GetDataItem(DataItemDetailEnum.ShooterType); //总榜 var RankingAll = TongJiListData[0]; ShooterList(RankingAll, eventId, season, ShooterEnum.AllShooter); //主场 var RankingMain = TongJiListData[1]; ShooterList(RankingMain, eventId, season, ShooterEnum.MainShooter); //客场 if (TongJiListData.Count > 2) { var RankingPassenger = TongJiListData[2]; ShooterList(RankingPassenger, eventId, season, ShooterEnum.PassengerShooter); } } public void ShooterList(HtmlNode ShooterAll, string eventId, string season, ShooterEnum shooterEnum) { HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(ShooterAll.InnerHtml); var allshooterList = doc1.DocumentNode.SelectNodes("//tbody/tr"); if (allshooterList == null || allshooterList.Count == 0) return; //读取射手榜信息并加入到数据库 foreach (var tritem in allshooterList) { try { HtmlDocument doc11 = new HtmlDocument(); doc11.LoadHtml(tritem.InnerHtml); var tdList = doc11.DocumentNode.SelectNodes("//td"); if (tdList == null || tdList.Count == 0) continue; F_ShooterList f_ShooterList = new F_ShooterList(); f_ShooterList.Id = Guid.NewGuid().ToString(); f_ShooterList.EventId = eventId; f_ShooterList.Season = season; f_ShooterList.ShooterType = DateItem.Where(o => o.ItemValue == ((int)shooterEnum).ToString()).Count() > 0 ? DateItem.Where(o => o.ItemValue == ((int)shooterEnum).ToString()).FirstOrDefault().Id : ""; f_ShooterList.Sort = tdList[0].InnerText.Trim() == null || tdList[0].InnerText.Trim() == "" ? 0 : int.Parse(tdList[0].InnerText.Trim().ToString()); var TeamName = tdList[1].InnerText.Trim().ToString(); var Team = TeamList.Where(o => o.Name == TeamName); f_ShooterList.TeamId = Team.Count() > 0 ? Team.FirstOrDefault().Id : ""; if (Team.Count() == 0) { f_ShooterList.Remark = TeamName; } f_ShooterList.PlayerName = tdList.Count() > 3 ? tdList[2].InnerText.Trim().ToString() : ""; var Player = PlayerList.Where(o => f_ShooterList.PlayerName != "" && o.ChineseName.Contains(f_ShooterList.PlayerName)); f_ShooterList.PlayerId = Player.Count() > 0 ? Player.FirstOrDefault().Id : ""; f_ShooterList.ContryName = tdList.Count() > 4 ? tdList[3].InnerText.Trim().ToString() : ""; var Contry = ContryList.Where(o => f_ShooterList.ContryName != "" && o.Name == f_ShooterList.ContryName); f_ShooterList.ContryId = Contry.Count() > 0 ? Contry.FirstOrDefault().Id : ""; f_ShooterList.Sum = tdList.Count() > 5 ? tdList[4].InnerText.Trim() == null || tdList[4].InnerText.Trim() == "" ? 0 : int.Parse(tdList[4].InnerText.Trim().ToString()) : 0; f_ShooterList.PenaltyKick = tdList.Count() > 6 ? tdList[5].InnerText.Trim() == null || tdList[5].InnerText.Trim() == "" ? 0 : int.Parse(tdList[5].InnerText.Trim().ToString()) : 0; f_ShooterList.HomeGoal = tdList.Count() > 7 ? tdList[6].InnerText.Trim() == null || tdList[6].InnerText.Trim() == "" ? 0 : int.Parse(tdList[6].InnerText.Trim().ToString()) : 0; f_ShooterList.PassengerGoal = tdList.Count() > 8 ? tdList[7].InnerText.Trim() == null || tdList[7].InnerText.Trim() == "" ? 0 : int.Parse(tdList[7].InnerText.Trim().ToString()) : 0; ShooterLists.Add(f_ShooterList); } catch (Exception ex) { throw; } } } #region SQL语句 /// /// 获取所有的赛事 /// private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null"; /// /// 获取最新两年的数据赛事 /// private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null and Season like '%{0}%'"; #endregion } }