using FCS.Common; using FCS.Interface; using FCS.Models; using FCS.Models.DTO; using FCS.Models.Entity; using HtmlAgilityPack; using Quartz; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; namespace FCS.Crawler.Basketball { /// /// 篮球联盟排名服务 /// public class B_LeagueRankingJob : CommonJob, IJob { private static List LeagueRankingList = new List(); private static List DateItem = new List(); private static List TeamList = new List(); public B_LeagueRankingJob() { log = new LogHelper(); services = IOC.Resolve(); } public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); } #region 更新数据 /// /// 获取所有数据 /// public void GetAll() { GetSqlString(GetAllEventUrl, ""); } /// /// 更新数据 /// /// public void Click(string Year = "") { if (Year == "") { Year = DateTime.Now.Year.ToString().Substring(2,2); } var sql = string.Format(GetNewEventUrl, Year); GetSqlString(sql, Year); } #endregion /// /// 解析数据 /// /// private void GetSqlString(string sqlstring, string Year) { ThreadPool.SetMinThreads(10, 10); ThreadPool.SetMaxThreads(500, 500); TeamList = services.Query().ToList(); LeagueRankingList = services.Query().ToList(); var ds = services.Query("and Remark is not null", "", "distinct EventId,Season,Remark").ToList(); if (Year != "") { ds = services.Query("and Remark is not null and Season like '%" + Year + "%'", "", "distinct EventId,Season,Remark").ToList(); } Task.Run(() => { foreach (var item in ds) { Task.Run(() => { var url = item.Remark.ToString().Replace("saiAll.do", "pai.do"); var eventId = item.EventId; var season = item.Season; GetData(url, eventId, season); }); } }); int maxWorkerThreads, workerThreads; int maxportThreads, portThreads; while (true) { /* GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。 */ ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads); ThreadPool.GetAvailableThreads(out workerThreads, out portThreads); Thread.Sleep(1000); Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads)); if (maxWorkerThreads - workerThreads == 0) { Console.WriteLine("Thread Finished!"); break; } } try { if (Year != "") { services.Delete(" and Season like '" + Year + "%'"); } services.SqlBulkCopyAdd(LeagueRankingList); } catch (Exception ex) { throw; } } /// /// 解析排行榜数据 /// /// /// /// private void GetData(string url, string eventId, string season) { HtmlParameterDTO dtomodel = new HtmlParameterDTO(); dtomodel.Url = url; HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel); if (doc.DocumentNode.InnerHtml == "Termination" || doc.DocumentNode.InnerHtml == "" || doc.DocumentNode.SelectNodes("//*[@class='paiming']") == null) { return; } var team = doc.DocumentNode.SelectNodes("//*[@class='paiming']"); foreach (var item in team) { HtmlDocument itemdoc = new HtmlDocument(); itemdoc.LoadHtml(item.InnerHtml); var trList = itemdoc.DocumentNode.SelectNodes("//table/tr").ToList(); var TypeName = ""; if (itemdoc.DocumentNode.InnerText.Contains("东部")) { TypeName = "东部排行"; } else { TypeName = "西部排行"; } trList.ForEach(p => { if (!p.InnerText.Contains(" 1) { B_LeagueRanking ranking = new B_LeagueRanking(); ranking.Id = Guid.NewGuid().ToString(); ranking.Sort = int.Parse(tdList[0].InnerText.ToString()); ranking.TeamId = TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).Count() > 0 ? TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).FirstOrDefault().Id : ""; ranking.EventId = eventId; ranking.Season = season; ranking.Win = int.Parse(tdList[2].InnerText.Trim().ToString()); ranking.Lose = int.Parse(tdList[3].InnerText.Trim().ToString()); ranking.WinRate = tdList[4].InnerText.Trim().ToString(); ranking.WinDivide = tdList[5].InnerText.Trim().ToString(); ranking.ScoreAvg = tdList[6].InnerText.Trim().ToString(); ranking.LossAvg = tdList[7].InnerText.Trim().ToString(); ranking.League = tdList[8].InnerText.Trim().ToString(); ranking.Division = tdList[9].InnerText.Trim().ToString(); ranking.Home = tdList[10].InnerText.Trim().ToString(); ranking.Visiting = tdList[11].InnerText.Trim().ToString(); ranking.NearlyTen = tdList[12].InnerText.Trim().ToString(); ranking.Status = tdList[13].InnerText.Trim().ToString(); ranking.Remark = TypeName; LeagueRankingList.Add(ranking); } } }); } } #region SQL语句 /// /// 获取所有的赛事 /// private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null"; /// /// 获取最新两年的数据赛事 /// private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null and Season like '%{0}%'"; #endregion } }