B_LeagueRankingJob.cs 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209
  1. using FCS.Common;
  2. using FCS.Interface;
  3. using FCS.Models;
  4. using FCS.Models.DTO;
  5. using FCS.Models.Entity;
  6. using HtmlAgilityPack;
  7. using Quartz;
  8. using System;
  9. using System.Collections.Generic;
  10. using System.Diagnostics;
  11. using System.Linq;
  12. using System.Text;
  13. using System.Threading;
  14. using System.Threading.Tasks;
  15. namespace FCS.Crawler.Basketball
  16. {
  17. /// <summary>
  18. /// 篮球联盟排名服务
  19. /// </summary>
  20. public class B_LeagueRankingJob : CommonJob, IJob
  21. {
  22. private static List<B_LeagueRanking> LeagueRankingList = new List<B_LeagueRanking>();
  23. private static List<DataItemDetail> DateItem = new List<DataItemDetail>();
  24. private static List<B_Team> TeamList = new List<B_Team>();
  25. public B_LeagueRankingJob()
  26. {
  27. log = new LogHelper();
  28. services = IOC.Resolve<IDTOpenCode>();
  29. }
  30. public void Execute(IJobExecutionContext context)
  31. {
  32. Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
  33. }
  34. #region 更新数据
  35. /// <summary>
  36. /// 获取所有数据
  37. /// </summary>
  38. public void GetAll()
  39. {
  40. GetSqlString(GetAllEventUrl, "");
  41. }
  42. /// <summary>
  43. /// 更新数据
  44. /// </summary>
  45. /// <param name="Year"></param>
  46. public void Click(string Year = "")
  47. {
  48. if (Year == "")
  49. {
  50. Year = DateTime.Now.Year.ToString().Substring(2,2);
  51. }
  52. var sql = string.Format(GetNewEventUrl, Year);
  53. GetSqlString(sql, Year);
  54. }
  55. #endregion
  56. /// <summary>
  57. /// 解析数据
  58. /// </summary>
  59. /// <param name="sqlstring"></param>
  60. private void GetSqlString(string sqlstring, string Year)
  61. {
  62. ThreadPool.SetMinThreads(10, 10);
  63. ThreadPool.SetMaxThreads(500, 500);
  64. TeamList = services.Query<B_Team>().ToList();
  65. LeagueRankingList = services.Query<B_LeagueRanking>().ToList();
  66. var ds = services.Query<B_Grouping>("and Remark is not null", "", "distinct EventId,Season,Remark").ToList();
  67. if (Year != "")
  68. {
  69. ds = services.Query<B_Grouping>("and Remark is not null and Season like '%" + Year + "%'", "", "distinct EventId,Season,Remark").ToList();
  70. }
  71. Task.Run(() =>
  72. {
  73. foreach (var item in ds)
  74. {
  75. Task.Run(() =>
  76. {
  77. var url = item.Remark.ToString().Replace("saiAll.do", "pai.do");
  78. var eventId = item.EventId;
  79. var season = item.Season;
  80. GetData(url, eventId, season);
  81. });
  82. }
  83. });
  84. int maxWorkerThreads, workerThreads;
  85. int maxportThreads, portThreads;
  86. while (true)
  87. {
  88. /*
  89. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  90. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  91. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  92. */
  93. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  94. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  95. Thread.Sleep(1000);
  96. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
  97. if (maxWorkerThreads - workerThreads == 0)
  98. {
  99. Console.WriteLine("Thread Finished!");
  100. break;
  101. }
  102. }
  103. try
  104. {
  105. if (Year != "")
  106. {
  107. services.Delete<B_LeagueRanking>(" and Season like '" + Year + "%'");
  108. }
  109. services.SqlBulkCopyAdd<B_LeagueRanking>(LeagueRankingList);
  110. }
  111. catch (Exception ex)
  112. {
  113. throw;
  114. }
  115. }
  116. /// <summary>
  117. /// 解析排行榜数据
  118. /// </summary>
  119. /// <param name="url"></param>
  120. /// <param name="eventId"></param>
  121. /// <param name="season"></param>
  122. private void GetData(string url, string eventId, string season)
  123. {
  124. HtmlParameterDTO dtomodel = new HtmlParameterDTO();
  125. dtomodel.Url = url;
  126. HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel);
  127. if (doc.DocumentNode.InnerHtml == "Termination" || doc.DocumentNode.InnerHtml == "" || doc.DocumentNode.SelectNodes("//*[@class='paiming']") == null)
  128. {
  129. return;
  130. }
  131. var team = doc.DocumentNode.SelectNodes("//*[@class='paiming']");
  132. foreach (var item in team)
  133. {
  134. HtmlDocument itemdoc = new HtmlDocument();
  135. itemdoc.LoadHtml(item.InnerHtml);
  136. var trList = itemdoc.DocumentNode.SelectNodes("//table/tr").ToList();
  137. var TypeName = "";
  138. if (itemdoc.DocumentNode.InnerText.Contains("东部"))
  139. {
  140. TypeName = "东部排行";
  141. }
  142. else
  143. {
  144. TypeName = "西部排行";
  145. }
  146. trList.ForEach(p =>
  147. {
  148. if (!p.InnerText.Contains("<th"))
  149. {
  150. HtmlDocument trdoc = new HtmlDocument();
  151. trdoc.LoadHtml(p.InnerHtml);
  152. if (doc.DocumentNode.SelectNodes("//td") == null)
  153. return;
  154. var tdList = trdoc.DocumentNode.InnerHtml.Contains("td") ? trdoc.DocumentNode.SelectNodes("//td").ToList() : null;
  155. if (tdList != null && tdList.Count() > 1)
  156. {
  157. B_LeagueRanking ranking = new B_LeagueRanking();
  158. ranking.Id = Guid.NewGuid().ToString();
  159. ranking.Sort = int.Parse(tdList[0].InnerText.ToString());
  160. ranking.TeamId = TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).Count() > 0 ? TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).FirstOrDefault().Id : "";
  161. ranking.EventId = eventId;
  162. ranking.Season = season;
  163. ranking.Win = int.Parse(tdList[2].InnerText.Trim().ToString());
  164. ranking.Lose = int.Parse(tdList[3].InnerText.Trim().ToString());
  165. ranking.WinRate = tdList[4].InnerText.Trim().ToString();
  166. ranking.WinDivide = tdList[5].InnerText.Trim().ToString();
  167. ranking.ScoreAvg = tdList[6].InnerText.Trim().ToString();
  168. ranking.LossAvg = tdList[7].InnerText.Trim().ToString();
  169. ranking.League = tdList[8].InnerText.Trim().ToString();
  170. ranking.Division = tdList[9].InnerText.Trim().ToString();
  171. ranking.Home = tdList[10].InnerText.Trim().ToString();
  172. ranking.Visiting = tdList[11].InnerText.Trim().ToString();
  173. ranking.NearlyTen = tdList[12].InnerText.Trim().ToString();
  174. ranking.Status = tdList[13].InnerText.Trim().ToString();
  175. ranking.Remark = TypeName;
  176. LeagueRankingList.Add(ranking);
  177. }
  178. }
  179. });
  180. }
  181. }
  182. #region SQL语句
  183. /// <summary>
  184. /// 获取所有的赛事
  185. /// </summary>
  186. private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null";
  187. /// <summary>
  188. /// 获取最新两年的数据赛事
  189. /// </summary>
  190. private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null and Season like '%{0}%'";
  191. #endregion
  192. }
  193. }