using System;
using System.Collections.Generic;
using FCS.Common;
using FCS.Interface;
using FCS.Models;
using HtmlAgilityPack;
using Quartz;
using System.Data;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Diagnostics;
namespace FCS.Crawler.ZCMatchRankingList
{
///
/// 射手榜排行
///
public class ShooterListJob : CommonJob, IJob
{
private static List ShooterLists = new List();
private static List TeamList = new List();
private static List PlayerList = new List();
private static List ContryList = new List();
private static List DateItem = new List();
public ShooterListJob()
{
log = new LogHelper();
services = IOC.Resolve();
}
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
GetAll();
}
#region 更新数据
///
/// 获取所有数据
///
public void GetAll()
{
GetSqlString(GetAllEventUrl, "");
}
///
/// 更新数据
///
///
public void Click(string Year = "")
{
if (Year == "")
{
Year = DateTime.Now.Year.ToString();
}
var sql = string.Format(GetNewEventUrl, Year);
GetSqlString(sql, Year);
}
#endregion
///
/// 解析数据
///
///
private void GetSqlString(string sqlstring, string Year)
{
ThreadPool.SetMinThreads(10, 10);
ThreadPool.SetMaxThreads(500, 500);
TeamList = services.GetTeamList();
PlayerList = services.GetPlayerList();
ContryList = services.GetContryList();
var ds = SqlHelper.ExecuteDataset(CommandType.Text, sqlstring);
if (ds != null && ds.Tables.Count > 0)
{
for (int i = 0; i < ds.Tables[0].Rows.Count; i++)
{
//拿去赛事组的链接
var url = ds.Tables[0].Rows[i]["Remark"].ToString();
var eventId = ds.Tables[0].Rows[i]["EventId"].ToString();
var season = ds.Tables[0].Rows[i]["Season"].ToString();
////获取到总进球单双的链接
Task.Run(() =>
{
HtmlDocument doc = CommonHelper.GetHtml(url, new Dictionary(), "", "", 10000, 100);
var TJList = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']");
if (TJList != null && TJList.Count() != 0 && TJList[0].InnerText.Contains("赛事统计"))
{
HtmlDocument doc1 = new HtmlDocument();
doc1.LoadHtml(TJList[0].InnerHtml);
var liList = doc1.DocumentNode.SelectNodes("//ul/li/a").ToList();
var Glod = liList.Where(o => o.InnerText.Contains("射手榜")).First();
if (Glod != null)
{
var shooturl = Glod.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value.ToString();
Task.Run(() =>
{
GetList(shooturl, eventId, season);
});
}
}
});
}
int maxWorkerThreads, workerThreads;
int maxportThreads, portThreads;
while (true)
{
/*
GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
*/
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
Thread.Sleep(1000);
Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
if (maxWorkerThreads - workerThreads == 0)
{
Console.WriteLine("Thread Finished!");
break;
}
}
try
{
if (Year != "")
{
services.Delete(" and Season like '%"+Year+"%'");
}
services.SqlBulkCopyAdd(ShooterLists);
Trace.WriteLine("F_ShooterList更新完毕");
}
catch (Exception ex)
{
throw;
}
}
}
///
/// 解析射手榜信息
///
///
///
///
public void GetList(string url, string eventId, string season)
{
if (url.Contains("wwaattssuunn"))
{
return;
}
HtmlDocument doc = CommonHelper.GetHtml(url, new Dictionary(), "", "", 10000, 100);
var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='zstab']");
if (TongJiListData == null || TongJiListData.Count() == 0) return;
DateItem = services.GetDataItem(DataItemDetailEnum.ShooterType);
//总榜
var RankingAll = TongJiListData[0];
ShooterList(RankingAll, eventId, season, ShooterEnum.AllShooter);
//主场
var RankingMain = TongJiListData[1];
ShooterList(RankingMain, eventId, season, ShooterEnum.MainShooter);
//客场
if (TongJiListData.Count > 2)
{
var RankingPassenger = TongJiListData[2];
ShooterList(RankingPassenger, eventId, season, ShooterEnum.PassengerShooter);
}
}
public void ShooterList(HtmlNode ShooterAll, string eventId, string season, ShooterEnum shooterEnum)
{
HtmlDocument doc1 = new HtmlDocument();
doc1.LoadHtml(ShooterAll.InnerHtml);
var allshooterList = doc1.DocumentNode.SelectNodes("//tbody/tr");
if (allshooterList == null || allshooterList.Count == 0) return;
//读取射手榜信息并加入到数据库
foreach (var tritem in allshooterList)
{
try
{
HtmlDocument doc11 = new HtmlDocument();
doc11.LoadHtml(tritem.InnerHtml);
var tdList = doc11.DocumentNode.SelectNodes("//td");
if (tdList == null || tdList.Count == 0) continue;
F_ShooterList f_ShooterList = new F_ShooterList();
f_ShooterList.Id = Guid.NewGuid().ToString();
f_ShooterList.EventId = eventId;
f_ShooterList.Season = season;
f_ShooterList.ShooterType = DateItem.Where(o => o.ItemValue == ((int)shooterEnum).ToString()).Count() > 0 ? DateItem.Where(o => o.ItemValue == ((int)shooterEnum).ToString()).FirstOrDefault().Id : "";
f_ShooterList.Sort = tdList[0].InnerText.Trim() == null || tdList[0].InnerText.Trim() == "" ? 0 : int.Parse(tdList[0].InnerText.Trim().ToString());
var TeamName = tdList[1].InnerText.Trim().ToString();
var Team = TeamList.Where(o => o.Name == TeamName);
f_ShooterList.TeamId = Team.Count() > 0 ? Team.FirstOrDefault().Id : "";
if (Team.Count() == 0)
{
f_ShooterList.Remark = TeamName;
}
f_ShooterList.PlayerName = tdList.Count() > 3 ? tdList[2].InnerText.Trim().ToString() : "";
var Player = PlayerList.Where(o => f_ShooterList.PlayerName != "" && o.ChineseName.Contains(f_ShooterList.PlayerName));
f_ShooterList.PlayerId = Player.Count() > 0 ? Player.FirstOrDefault().Id : "";
f_ShooterList.ContryName = tdList.Count() > 4 ? tdList[3].InnerText.Trim().ToString() : "";
var Contry = ContryList.Where(o => f_ShooterList.ContryName != "" && o.Name == f_ShooterList.ContryName);
f_ShooterList.ContryId = Contry.Count() > 0 ? Contry.FirstOrDefault().Id : "";
f_ShooterList.Sum = tdList.Count() > 5 ? tdList[4].InnerText.Trim() == null || tdList[4].InnerText.Trim() == "" ? 0 : int.Parse(tdList[4].InnerText.Trim().ToString()) : 0;
f_ShooterList.PenaltyKick = tdList.Count() > 6 ? tdList[5].InnerText.Trim() == null || tdList[5].InnerText.Trim() == "" ? 0 : int.Parse(tdList[5].InnerText.Trim().ToString()) : 0;
f_ShooterList.HomeGoal = tdList.Count() > 7 ? tdList[6].InnerText.Trim() == null || tdList[6].InnerText.Trim() == "" ? 0 : int.Parse(tdList[6].InnerText.Trim().ToString()) : 0;
f_ShooterList.PassengerGoal = tdList.Count() > 8 ? tdList[7].InnerText.Trim() == null || tdList[7].InnerText.Trim() == "" ? 0 : int.Parse(tdList[7].InnerText.Trim().ToString()) : 0;
ShooterLists.Add(f_ShooterList);
}
catch (Exception ex)
{
throw;
}
}
}
#region SQL语句
///
/// 获取所有的赛事
///
private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null";
///
/// 获取最新两年的数据赛事
///
private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null and Season like '%{0}%'";
#endregion
}
}