using System;
using System.Collections.Generic;
using FCS.Common;
using FCS.Crawler.Tools;
using FCS.Interface;
using FCS.Models;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Quartz;
using System.Data;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Diagnostics;
using FCS.Models.DTO;
namespace FCS.Crawler.ZCLotteryTeam
{
///
/// 球員爬取
///
public class FootballPlayerJob : CommonJob, IJob
{
public List relations = new List();//球员球队关系
public List players = new List();//球员列表
public List allPlayer = new List();
public FootballPlayerJob()
{
log = new LogHelper();
services = IOC.Resolve();
}
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
GetAll();
}
public void GetAll()
{
ThreadPool.SetMinThreads(10, 10);
ThreadPool.SetMaxThreads(100, 100);
var ds = SqlHelper.ExecuteDataset(CommandType.Text, GetAllTeamUrl);
allPlayer = services.GetPlayerList();
if (ds != null && ds.Tables.Count > 0)
{
//获取赛事存取的url ds.Tables[0].Rows.Count
for (int i = 0; i < ds.Tables[0].Rows.Count; i++)
{
//异步加载分组
var url = ds.Tables[0].Rows[i]["Remark"].ToString();
var TeamId = ds.Tables[0].Rows[i]["Id"].ToString();
var TeamName = ds.Tables[0].Rows[i]["Name"].ToString();
Task.Run(() =>
{
Trace.WriteLine("个数:" + i);
GetPlayerListHtml(url, TeamId, TeamName);
});
}
int maxWorkerThreads, workerThreads;
int maxportThreads, portThreads;
while (true)
{
/*
GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
*/
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
Thread.Sleep(1000);
Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
if (maxWorkerThreads - workerThreads == 0)
{
Console.WriteLine("Thread Finished!");
break;
}
}
}
try
{
services.SqlBulkCopyAdd(players);
services.SqlBulkCopyAdd(relations);
}
catch (Exception ex)
{
throw;
}
}
///
/// 读取球员列表
///
///
///
///
///
public bool GetPlayerListHtml(string url, string teamId, string teamName)
{
if (url.Contains("wwaattssuunn"))
{
return false;
}
HtmlParameterDTO dtomodel = new HtmlParameterDTO();
dtomodel.Url = url;
dtomodel.Timeout = 10 * 1000;
dtomodel.IsCheckEmpty = false;
HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel);// CommonHelper.GetHtml(url, new Dictionary(), "", "", 90000, 10);
if (doc.DocumentNode.InnerHtml == "Termination")
{
return false;
}
//获取球员的列表
var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='mingd_right']/dl");
if (TongJiListData == null || TongJiListData.Count == 0) return false;
foreach (var list in TongJiListData)
{
HtmlDocument doc1 = new HtmlDocument();
doc1.LoadHtml(list.InnerHtml);
var playerLIst = doc1.DocumentNode.SelectNodes("//dd/a");
var playeerType = doc1.DocumentNode.SelectNodes("//dt");
var position = "";
if (playeerType.Count > 0)
{
position = playeerType.FirstOrDefault().InnerText.Replace("/t", "").Trim();
}
if (playerLIst == null)
{
continue;
}
Task.Run(() =>
{
foreach (var playitem in playerLIst)
{
var playerurl = playitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
var playerName = playitem.InnerText.Trim();
var playerInfo = "";
if (allPlayer.Where(o => o.ChineseName == playerName).Count() == 0)
{
playerInfo = GetPlayerInfo(playerurl, playerName);
}
var playid = ""; var num = "";
if (playerInfo != "")//不为空表示需要新增
{
playid = playerInfo.Split(',').Count() > 0 ? playerInfo.Split(',')[0] : "";
num = playerInfo.Split(',').Count() > 1 ? playerInfo.Split(',')[1] : "0";
F_Relation f_Relation = new F_Relation();
f_Relation.Id = Guid.NewGuid().ToString();
f_Relation.TeamId = teamId;
f_Relation.PlayerId = playid;
f_Relation.PoloShirt = num == "" ? 0 : int.Parse(num);
f_Relation.Position = position;
f_Relation.CreateDateTime = DateTime.Now;
f_Relation.ContractTime = DateTime.Now;
if (f_Relation.PlayerId != null && f_Relation.PlayerId != "")
{
relations.Add(f_Relation);
}
}
}
});
}
return true;
}
///
/// 解析球员信息,并返回id
///
///
///
///
public string GetPlayerInfo(string url, string name)
{
var mainUrl = "http://saishi.zgzcw.com/" + url;
HtmlParameterDTO dtomodel = new HtmlParameterDTO();
dtomodel.Url = url;
dtomodel.Timeout = 10 * 1000;
dtomodel.IsCheckEmpty = false;
HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel); //CommonHelper.GetHtml(mainUrl,new Dictionary() ,"","",90000,10);
if (doc.DocumentNode.InnerHtml== "Termination")
{
return "";
}
//球员图片
var teamImg = doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dt/img");
//球员介绍
var teamIntroData = doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']") == null || doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']").Count == 0 ? "" : doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']").FirstOrDefault().InnerHtml.Trim();
//球员的基本信息
if (doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd") == null || doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd").Count == 0)
{
return "";
}
var playerData = doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd").ToList();
F_Players f_Players = new F_Players();
f_Players.Id = Guid.NewGuid().ToString();
f_Players.ChineseName = name;
f_Players.Describe = teamIntroData;
f_Players.Remark = mainUrl;
f_Players.LogoImage = teamImg.FirstOrDefault().Attributes.SingleOrDefault(a => a.Name.Equals("src")).Value;
var contry = playerData.Where(o => o.InnerText.Contains("国家")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("国家")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null;
f_Players.Nationality = contry != null && contry.Count() > 1 ? contry[1].Trim() : "";
var Weight = playerData.Where(o => o.InnerText.Contains("体重")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("体重")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null;
f_Players.Weight = Weight != null && Weight.Count() > 1 ? Weight[1].Trim() : "";
var Stature = playerData.Where(o => o.InnerText.Contains("身高")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("身高")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null;
f_Players.Stature = Stature != null && Stature.Count() > 1 ? Stature[1].Trim() : "";
var Birthday = playerData.Where(o => o.InnerText.Contains("生日")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("生日")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null;
f_Players.Birthday = Birthday != null && Birthday.Count() > 1 ? Birthday[1].Trim() : "";
var num = playerData.Where(o => o.InnerText.Contains("号码")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("号码")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null;
var playerNum = num != null && num.Count() > 1 ? num[1].Trim() : "";
//if (allPlayer.Where(o => o.ChineseName == f_Players.ChineseName).Count() == 0)
//{
players.Add(f_Players);
return f_Players.Id + "," + playerNum;
//}
//services.AddPlayers(FCSLottery.F_Players, f_Players);
//return "";
}
#region SQL语句
private static string GetAllTeamUrl = @"select Id,Name, Remark from F_Team where Remark is not null";
#endregion
}
}