B_SizeBallRankingJob.cs 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183
  1. using System;
  2. using FCS.Common;
  3. using FCS.Interface;
  4. using FCS.Models;
  5. using FCS.Models.Entity;
  6. using HtmlAgilityPack;
  7. using Quartz;
  8. using System;
  9. using System.Collections.Generic;
  10. using System.Diagnostics;
  11. using System.Linq;
  12. using System.Text;
  13. using System.Threading;
  14. using System.Threading.Tasks;
  15. using FCS.Models.DTO;
  16. namespace FCS.Crawler.Basketball
  17. {
  18. /// <summary>
  19. /// 篮球大小盘路
  20. /// </summary>
  21. public class B_SizeBallRankingJob : CommonJob, IJob
  22. {
  23. private static List<B_SizeBallRanking> SizeBallRankingList = new List<B_SizeBallRanking>();
  24. private static List<DataItemDetail> DateItem = new List<DataItemDetail>();
  25. private static List<B_Team> TeamList = new List<B_Team>();
  26. public B_SizeBallRankingJob()
  27. {
  28. }
  29. public void Execute(IJobExecutionContext context)
  30. {
  31. Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
  32. }
  33. #region 更新数据
  34. /// <summary>
  35. /// 获取所有数据
  36. /// </summary>
  37. public void GetAll()
  38. {
  39. GetSqlString("");
  40. }
  41. /// <summary>
  42. /// 更新数据
  43. /// </summary>
  44. /// <param name="Year"></param>
  45. public void Click(string Year = "")
  46. {
  47. if (Year == "")
  48. {
  49. Year = DateTime.Now.Year.ToString().Substring(2, 2); ;
  50. }
  51. GetSqlString(Year);
  52. }
  53. #endregion
  54. /// <summary>
  55. /// 解析数据
  56. /// </summary>
  57. /// <param name="sqlstring"></param>
  58. private void GetSqlString(string Year)
  59. {
  60. ThreadPool.SetMinThreads(10, 10);
  61. ThreadPool.SetMaxThreads(500, 500);
  62. TeamList = services.Query<B_Team>().ToList();
  63. SizeBallRankingList = services.Query<B_SizeBallRanking>().ToList();
  64. var dataItem = services.GetDataItem(DataItemDetailEnum.FootBallSizeBallType);
  65. var ds = services.Query<B_Grouping>("and Remark is not null", "", "distinct EventId,Season,Remark").ToList();
  66. if (Year != "")
  67. {
  68. ds = services.Query<B_Grouping>("and Remark is not null and Season like '%" + Year + "%'", "", "distinct EventId,Season,Remark").ToList();
  69. }
  70. Task.Run(() =>
  71. {
  72. foreach (var item in ds)
  73. {
  74. Task.Run(() =>
  75. {
  76. var eventId = item.EventId;
  77. var season = item.Season;
  78. var url = item.Remark.ToString().Replace("saiAll.do", "1/bigSmallBall.do");
  79. GetData(url, eventId, season, dataItem.Where(o => o.ItemValue == "1").First().Id.ToString());
  80. url = item.Remark.ToString().Replace("saiAll.do", "2/bigSmallBall.do");
  81. GetData(url, eventId, season, dataItem.Where(o => o.ItemValue == "2").First().Id.ToString());
  82. url = item.Remark.ToString().Replace("saiAll.do", "3/bigSmallBall.do");
  83. GetData(url, eventId, season, dataItem.Where(o => o.ItemValue == "3").First().Id.ToString());
  84. });
  85. }
  86. });
  87. int maxWorkerThreads, workerThreads;
  88. int maxportThreads, portThreads;
  89. while (true)
  90. {
  91. /*
  92. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  93. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  94. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  95. */
  96. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  97. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  98. Thread.Sleep(1000);
  99. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
  100. if (maxWorkerThreads - workerThreads == 0)
  101. {
  102. Console.WriteLine("Thread Finished!");
  103. break;
  104. }
  105. }
  106. try
  107. {
  108. if (Year != "")
  109. {
  110. services.Delete<B_SizeBallRanking>(" and Season like '" + Year + "%'");
  111. }
  112. services.SqlBulkCopyAdd<B_SizeBallRanking>(SizeBallRankingList);
  113. Trace.WriteLine("B_SizeBallRanking更新完毕");
  114. }
  115. catch (Exception ex)
  116. {
  117. throw;
  118. }
  119. }
  120. /// <summary>
  121. /// 解析排行榜数据
  122. /// </summary>
  123. /// <param name="url"></param>
  124. /// <param name="eventId"></param>
  125. /// <param name="season"></param>
  126. private void GetData(string url, string eventId, string season, string sizeBallType)
  127. {
  128. HtmlParameterDTO dtomodel = new HtmlParameterDTO();
  129. dtomodel.Url = url;
  130. HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel);
  131. if (doc.DocumentNode.InnerHtml == "Termination" || doc.DocumentNode.InnerHtml == "" || doc.DocumentNode.SelectNodes("//*[@class='paiming']") == null)
  132. {
  133. return;
  134. }
  135. var team = doc.DocumentNode.SelectNodes("//*[@class='paiming']").First();
  136. HtmlDocument itemdoc = new HtmlDocument();
  137. itemdoc.LoadHtml(team.InnerHtml);
  138. var trList = itemdoc.DocumentNode.SelectNodes("//table/tr").ToList();
  139. trList.ForEach(p =>
  140. {
  141. if (!p.InnerHtml.Contains("<th"))
  142. {
  143. HtmlDocument trdoc = new HtmlDocument();
  144. trdoc.LoadHtml(p.InnerHtml);
  145. var tdList = trdoc.DocumentNode.InnerHtml.Contains("td") ? trdoc.DocumentNode.SelectNodes("//td").ToList() : null;
  146. if (tdList != null && tdList.Count() > 1)
  147. {
  148. B_SizeBallRanking sizing = new B_SizeBallRanking();
  149. sizing.Id = Guid.NewGuid().ToString();
  150. sizing.Sort = int.Parse(tdList[0].InnerText.Trim().ToString());
  151. sizing.SizeBallType = sizeBallType;
  152. sizing.TeamId = TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).Count() > 0 ? TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).FirstOrDefault().Id : "";
  153. sizing.EventId = eventId;
  154. sizing.Season = season;
  155. sizing.Sum = int.Parse(tdList[2].InnerText.Trim().ToString());
  156. sizing.LargeNum = int.Parse(tdList[3].InnerText.Trim().ToString());
  157. sizing.GoNum = int.Parse(tdList[4].InnerText.Trim().ToString());
  158. sizing.SmallNum = int.Parse(tdList[5].InnerText.Trim().ToString());
  159. sizing.LargePercentage = tdList[6].InnerText.Trim().ToString();
  160. sizing.GoPercentage = tdList[7].InnerText.Trim().ToString();
  161. sizing.SmallPercentage = tdList[8].InnerText.Trim().ToString();
  162. SizeBallRankingList.Add(sizing);
  163. }
  164. }
  165. });
  166. }
  167. }
  168. }