using System;
using System.Collections.Generic;
using FCS.Common;
using FCS.Interface;
using FCS.Models;
using HtmlAgilityPack;
using Quartz;
using System.Data;
using System.Linq;
using System.Threading.Tasks;
using System.Threading;
using System.Diagnostics;
namespace FCS.Crawler.ZCMatchRankingList
{
///
/// 进球数单双统计
///
public class GoalSingleDbListJob : CommonJob, IJob
{
private static List SingleDoubleLists = new List();
private static List TeamList = new List();
public GoalSingleDbListJob()
{
log = new LogHelper();
services = IOC.Resolve();
}
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
GetAll();
}
#region 更新数据
///
/// 获取所有数据
///
public void GetAll()
{
GetSqlString(GetAllEventUrl,"");
}
///
/// 更新数据,默认为今年数据
///
///
public void Click(string Year = "")
{
if (Year == "")
{
Year = DateTime.Now.Year.ToString();
}
var sql = string.Format(GetNewEventUrl, Year);
GetSqlString(sql, Year);
}
#endregion
///
/// 解析数据
///
///
private void GetSqlString(string sqlstring,string Year)
{
ThreadPool.SetMinThreads(10, 10);
ThreadPool.SetMaxThreads(500, 500);
TeamList = services.GetTeamList();
var ds = SqlHelper.ExecuteDataset(CommandType.Text, sqlstring);
if (ds != null && ds.Tables.Count > 0)
{
for (int i = 0; i < ds.Tables[0].Rows.Count; i++)
{
//拿去赛事组的链接
var url = ds.Tables[0].Rows[i]["Remark"].ToString();
var eventId = ds.Tables[0].Rows[i]["EventId"].ToString();
var season = ds.Tables[0].Rows[i]["Season"].ToString();
////获取到总进球单双的链接
Task.Run(() =>
{
HtmlDocument doc = CommonHelper.GetHtml(url, new Dictionary(), "", "", 90000, 10);
var TJList = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']");
if (TJList != null && TJList.Count() != 0 && TJList[0].InnerText.Contains("赛事统计"))
{
HtmlDocument doc1 = new HtmlDocument();
doc1.LoadHtml(TJList[0].InnerHtml);
var liList = doc1.DocumentNode.SelectNodes("//ul/li/a").ToList();
var Glod = liList.Where(o => o.InnerText.Contains("进球单双")).First();
if (Glod != null)
{
var jqdsurl = Glod.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value.ToString();
Task.Run(() =>
{
GetJQDSHtml(jqdsurl, eventId, season);
});
}
}
});
}
int maxWorkerThreads, workerThreads;
int maxportThreads, portThreads;
while (true)
{
/*
GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
*/
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
Thread.Sleep(1000);
Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
if (maxWorkerThreads - workerThreads == 0)
{
Console.WriteLine("Thread Finished!");
break;
}
}
try
{
if (Year!="")
{
services.Delete(" and Season like '"+Year+"%'");
}
services.SqlBulkCopyAdd(SingleDoubleLists);
Trace.WriteLine("F_SingleDoubleList更新完毕");
}
catch (Exception ex)
{
throw;
}
}
}
///
/// 解析进球单双页面
///
///
///
///
public void GetJQDSHtml(string url, string eventId, string season)
{
if (url.Contains("wwaattssuunn"))
{
return;
}
HtmlDocument doc = CommonHelper.GetHtml(url,new Dictionary(),"","",9000,10);
var trData = doc.DocumentNode.SelectNodes("//*[@class='table_out']/table/tbody/tr");
if (trData == null || trData.Count() == 0) return;
//读取进球单双信息并加入到数据库
foreach (var tritem in trData)
{
try
{
HtmlDocument doc11 = new HtmlDocument();
doc11.LoadHtml(tritem.InnerHtml);
var tdList = doc11.DocumentNode.SelectNodes("//td");
if (tdList == null || tdList.Count == 0) continue;
F_SingleDoubleList f_SingleDoubleList = new F_SingleDoubleList();
f_SingleDoubleList.Id = Guid.NewGuid().ToString();
f_SingleDoubleList.EventId = eventId;
f_SingleDoubleList.Season = season;
f_SingleDoubleList.Sort = tdList[0].InnerText.Trim() == null ? 0 : int.Parse(tdList[0].InnerText.Trim().ToString());
var TeamName = tdList[1].InnerText.Trim().ToString();
var Team = TeamList.Where(o => o.Name == TeamName);
f_SingleDoubleList.TeamId = Team.Count() > 0 ? Team.FirstOrDefault().Id : "";
if (Team.Count() == 0)
{
f_SingleDoubleList.Remark = TeamName;
}
f_SingleDoubleList.OneNum = tdList.Count()>3? int.Parse(tdList[2].InnerText.Trim().ToString()):0;
f_SingleDoubleList.ZeroNum = tdList.Count() > 4 ? int.Parse(tdList[3].InnerText.Trim().ToString()) : 0;
f_SingleDoubleList.TwoNum = tdList.Count() > 5 ? int.Parse(tdList[4].InnerText.Trim().ToString()) : 0;
f_SingleDoubleList.ThreeNum = tdList.Count() > 6 ? int.Parse(tdList[5].InnerText.Trim().ToString()) : 0;
f_SingleDoubleList.FourNum = tdList.Count() > 7 ? int.Parse(tdList[6].InnerText.Trim().ToString()) : 0;
f_SingleDoubleList.OddNum = tdList.Count() > 8 ? int.Parse(tdList[7].InnerText.Trim().ToString()) : 0;
f_SingleDoubleList.EvenNum = tdList.Count() > 9 ? int.Parse(tdList[8].InnerText.Trim().ToString()) : 0;
SingleDoubleLists.Add(f_SingleDoubleList);
}
catch (Exception ex)
{
throw;
}
}
}
#region SQL语句
///
/// 获取所有的赛事
///
private static string GetAllEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null";
///
/// 获取最新两年的数据赛事
///
private static string GetNewEventUrl = @"select distinct EventId,Season,Remark from F_Grouping where Remark is not null and Season like '%{0}%'";
private static string DeleteNewEventUrl = @"delete F_SingleDoubleList where Season like '{0}%'";
#endregion
}
}