using FCS.Common;
using FCS.Interface;
using FCS.Models;
using FCS.Models.DTO;
using FCS.Models.Entity;
using HtmlAgilityPack;
using Quartz;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace FCS.Crawler.Basketball
{
///
/// 篮球联盟排名服务
///
public class B_LeagueRankingJob : CommonJob, IJob
{
private static List LeagueRankingList = new List();
private static List DateItem = new List();
private static List TeamList = new List();
public B_LeagueRankingJob()
{
log = new LogHelper();
services = IOC.Resolve();
}
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
}
#region 更新数据
///
/// 获取所有数据
///
public void GetAll()
{
GetSqlString(GetAllEventUrl, "");
}
///
/// 更新数据
///
///
public void Click(string Year = "")
{
if (Year == "")
{
Year = DateTime.Now.Year.ToString().Substring(2,2);
}
var sql = string.Format(GetNewEventUrl, Year);
GetSqlString(sql, Year);
}
#endregion
///
/// 解析数据
///
///
private void GetSqlString(string sqlstring, string Year)
{
ThreadPool.SetMinThreads(10, 10);
ThreadPool.SetMaxThreads(500, 500);
TeamList = services.Query().ToList();
LeagueRankingList = services.Query().ToList();
var ds = services.Query("and Remark is not null", "", "distinct EventId,Season,Remark").ToList();
if (Year != "")
{
ds = services.Query("and Remark is not null and Season like '%" + Year + "%'", "", "distinct EventId,Season,Remark").ToList();
}
Task.Run(() =>
{
foreach (var item in ds)
{
Task.Run(() =>
{
var url = item.Remark.ToString().Replace("saiAll.do", "pai.do");
var eventId = item.EventId;
var season = item.Season;
GetData(url, eventId, season);
});
}
});
int maxWorkerThreads, workerThreads;
int maxportThreads, portThreads;
while (true)
{
/*
GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
*/
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
Thread.Sleep(1000);
Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
if (maxWorkerThreads - workerThreads == 0)
{
Console.WriteLine("Thread Finished!");
break;
}
}
try
{
if (Year != "")
{
services.Delete(" and Season like '" + Year + "%'");
}
services.SqlBulkCopyAdd(LeagueRankingList);
}
catch (Exception ex)
{
throw;
}
}
///
/// 解析排行榜数据
///
///
///
///
private void GetData(string url, string eventId, string season)
{
HtmlParameterDTO dtomodel = new HtmlParameterDTO();
dtomodel.Url = url;
HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel);
if (doc.DocumentNode.InnerHtml == "Termination" || doc.DocumentNode.InnerHtml == "" || doc.DocumentNode.SelectNodes("//*[@class='paiming']") == null)
{
return;
}
var team = doc.DocumentNode.SelectNodes("//*[@class='paiming']");
foreach (var item in team)
{
HtmlDocument itemdoc = new HtmlDocument();
itemdoc.LoadHtml(item.InnerHtml);
var trList = itemdoc.DocumentNode.SelectNodes("//table/tr").ToList();
var TypeName = "";
if (itemdoc.DocumentNode.InnerText.Contains("东部"))
{
TypeName = "东部排行";
}
else
{
TypeName = "西部排行";
}
trList.ForEach(p =>
{
if (!p.InnerText.Contains(" 1)
{
B_LeagueRanking ranking = new B_LeagueRanking();
ranking.Id = Guid.NewGuid().ToString();
ranking.Sort = int.Parse(tdList[0].InnerText.ToString());
ranking.TeamId = TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).Count() > 0 ? TeamList.Where(o => o.Name == tdList[1].InnerText.Trim().ToString()).FirstOrDefault().Id : "";
ranking.EventId = eventId;
ranking.Season = season;
ranking.Win = int.Parse(tdList[2].InnerText.Trim().ToString());
ranking.Lose = int.Parse(tdList[3].InnerText.Trim().ToString());
ranking.WinRate = tdList[4].InnerText.Trim().ToString();
ranking.WinDivide = tdList[5].InnerText.Trim().ToString();
ranking.ScoreAvg = tdList[6].InnerText.Trim().ToString();
ranking.LossAvg = tdList[7].InnerText.Trim().ToString();
ranking.League = tdList[8].InnerText.Trim().ToString();
ranking.Division = tdList[9].InnerText.Trim().ToString();
ranking.Home = tdList[10].InnerText.Trim().ToString();
ranking.Visiting = tdList[11].InnerText.Trim().ToString();
ranking.NearlyTen = tdList[12].InnerText.Trim().ToString();
ranking.Status = tdList[13].InnerText.Trim().ToString();
ranking.Remark = TypeName;
LeagueRankingList.Add(ranking);
}
}
});
}
}
#region SQL语句
///
/// 获取所有的赛事
///
private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null";
///
/// 获取最新两年的数据赛事
///
private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from B_Grouping where Remark is not null and Season like '%{0}%'";
#endregion
}
}
|