123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209 |
- using FCS.Common;
- using FCS.Interface;
- using FCS.Models;
- using FCS.Models.DTO;
- using FCS.Models.Entity;
- using HtmlAgilityPack;
- using Quartz;
- using System;
- using System.Collections.Generic;
- using System.Diagnostics;
- using System.Linq;
- using System.Text;
- using System.Threading;
- using System.Threading.Tasks;
- namespace FCS.Crawler.Basketball
- {
- /// <summary>
- /// 篮球联盟排名服务
- /// </summary>
- public class B_LeagueRankingJob : CommonJob, IJob
- {
- private static List<B_LeagueRanking> LeagueRankingList = new List<B_LeagueRanking>();
- private static List<DataItemDetail> DateItem = new List<DataItemDetail>();
- private static List<B_Team> TeamList = new List<B_Team>();
- public B_LeagueRankingJob()
- {
- log = new LogHelper();
- services = IOC.Resolve<IDTOpenCode>();
- }
- public void Execute(IJobExecutionContext context)
- {
- Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
- }
- #region 更新数据
- /// <summary>
- /// 获取所有数据
- /// </summary>
- public void GetAll()
- {
- GetSqlString(GetAllEventUrl, "");
- }
- /// <summary>
- /// 更新数据
- /// </summary>
- /// <param name="Year"></param>
- public void Click(string Year = "")
- {
- if (Year == "")
- {
- Year = DateTime.Now.Year.ToString().Substring(2,2);
- }
- var sql = string.Format(GetNewEventUrl, Year);
- GetSqlString(sql, Year);
- }
- #endregion
- /// <summary>
- /// 解析数据
- /// </summary>
- /// <param name="sqlstring"></param>
- private void GetSqlString(string sqlstring, string Year)
- {
- ThreadPool.SetMinThreads(10, 10);
- ThreadPool.SetMaxThreads(500, 500);
- TeamList = services.Query<B_Team>().ToList();
- LeagueRankingList = services.Query<B_LeagueRanking>().ToList();
- var ds = services.Query<B_Grouping>("and Remark is not null", "", "distinct EventId,Season,Remark").ToList();
- if (Year != "")
- {
- ds = services.Query<B_Grouping>("and Remark is not null and Season like '%" + Year + "%'", "", "distinct EventId,Season,Remark").ToList();
- }
- Task.Run(() =>
- {
- foreach (var item in ds)
- {
- Task.Run(() =>
- {
- var url = item.Remark.ToString().Replace("saiAll.do", "pai.do");
- var eventId = item.EventId;
- var season = item.Season;
- GetData(url, eventId, season);
- });
- }
- });
- int maxWorkerThreads, workerThreads;
- int maxportThreads, portThreads;
- while (true)
- {
- /*
- GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
- 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
- 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
- */
- ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
- ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
- Thread.Sleep(1000);
- Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
- if (maxWorkerThreads - workerThreads == 0)
- {
- Console.WriteLine("Thread Finished!");
- break;
- }
- }
- try
- {
- if (Year != "")
- {
- services.Delete<B_LeagueRanking>(" and Season like '" + Year + "%'");
- }
- services.SqlBulkCopyAdd<B_LeagueRanking>(LeagueRankingList);
- }
- catch (Exception ex)
- {
- throw;
- }
- }
- /// <summary>
- /// 解析排行榜数据
- /// </summary>
- /// <param name="url"></param>
- /// <param name="eventId"></param>
- /// <param name="season"></param>
- private void GetData(string url, string eventId, string season)
- {
- HtmlParameterDTO dtomodel = new HtmlParameterDTO();
- dtomodel.Url = url;
- HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel);
- if (doc.DocumentNode.InnerHtml == "Termination" || doc.DocumentNode.InnerHtml == "" || doc.DocumentNode.SelectNodes("//*[@class='paiming']") == null)
- {
- return;
- }
- var team = doc.DocumentNode.SelectNodes("//*[@class='paiming']");
- foreach (var item in team)
- {
- HtmlDocument itemdoc = new HtmlDocument();
- itemdoc.LoadHtml(item.InnerHtml);
- var trList = itemdoc.DocumentNode.SelectNodes("//table/tr").ToList();
- var TypeName = "";
- if (itemdoc.DocumentNode.InnerText.Contains("东部"))
- {
- TypeName = "东部排行";
- }
- else
- {
- TypeName = "西部排行";
- }
- trList.ForEach(p =>
- {
- if (!p.InnerText.Contains("<th"))
- {
- HtmlDocument trdoc = new HtmlDocument();
- trdoc.LoadHtml(p.InnerHtml);
- if (doc.DocumentNode.SelectNodes("//td") == null)
- return;
- var tdList = trdoc.DocumentNode.InnerHtml.Contains("td") ? trdoc.DocumentNode.SelectNodes("//td").ToList() : null;
- if (tdList != null && tdList.Count() > 1)
- {
- B_LeagueRanking ranking = new B_LeagueRanking();
- ranking.Id = Guid.NewGuid().ToString();
- ranking.Sort = int.Parse(tdList[0].InnerText.ToString());
- ranking.TeamId = TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).Count() > 0 ? TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).FirstOrDefault().Id : "";
- ranking.EventId = eventId;
- ranking.Season = season;
- ranking.Win = int.Parse(tdList[2].InnerText.Trim().ToString());
- ranking.Lose = int.Parse(tdList[3].InnerText.Trim().ToString());
- ranking.WinRate = tdList[4].InnerText.Trim().ToString();
- ranking.WinDivide = tdList[5].InnerText.Trim().ToString();
- ranking.ScoreAvg = tdList[6].InnerText.Trim().ToString();
- ranking.LossAvg = tdList[7].InnerText.Trim().ToString();
- ranking.League = tdList[8].InnerText.Trim().ToString();
- ranking.Division = tdList[9].InnerText.Trim().ToString();
- ranking.Home = tdList[10].InnerText.Trim().ToString();
- ranking.Visiting = tdList[11].InnerText.Trim().ToString();
- ranking.NearlyTen = tdList[12].InnerText.Trim().ToString();
- ranking.Status = tdList[13].InnerText.Trim().ToString();
- ranking.Remark = TypeName;
- LeagueRankingList.Add(ranking);
- }
- }
- });
- }
- }
- #region SQL语句
- /// <summary>
- /// 获取所有的赛事
- /// </summary>
- private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null";
- /// <summary>
- /// 获取最新两年的数据赛事
- /// </summary>
- private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null and Season like '%{0}%'";
- #endregion
- }
- }
|