using System; using System.Collections.Generic; using FCS.Common; using FCS.Crawler.Tools; using FCS.Interface; using FCS.Models; using HtmlAgilityPack; using Newtonsoft.Json; using Quartz; using System.Data; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Diagnostics; using FCS.Models.DTO; namespace FCS.Crawler.ZCLotteryTeam { /// <summary> /// 球員爬取 /// </summary> public class FootballPlayerJob : CommonJob, IJob { public List<F_Relation> relations = new List<F_Relation>();//球员球队关系 public List<F_Players> players = new List<F_Players>();//球员列表 public List<F_Players> allPlayer = new List<F_Players>(); F_Players g; public FootballPlayerJob() { log = new LogHelper(); services = IOC.Resolve<IDTOpenCode>(); g = new F_Players(); } public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); GetAll(); } public void GetAll() { var ds = services.Query<F_Team>("and Remark is not null").ToList(); allPlayer = services.GetPlayerList(); if (ds != null && ds.Count > 0) { //获取赛事存取的url ds.Tables[0].Rows.Count //int max = ds.Count; //int num = 0; ds.ForEach(p => { //异步加载分组 var url = p.Remark.ToString(); var TeamId = p.Id.ToString(); var TeamName = p.Name.ToString(); Task.Run(() => { GetPlayerListHtml(url, TeamId, TeamName); }); //lock (g) //{ // num++; // Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 //} }); //lock (g) //{ // while (num < max) // { // Monitor.Wait(g);//等待 // } //} //Trace.WriteLine("获取球员 完结"); while (true) { if (CommonHelper.ThreadsFinsh()) break; } } try { services.SqlBulkCopyAdd<F_Players>(players); services.SqlBulkCopyAdd<F_Relation>(relations); } catch (Exception ex) { throw; } } /// <summary> /// 读取球员列表 /// </summary> /// <param name="url"></param> /// <param name="teamId"></param> /// <param name="teamName"></param> /// <returns></returns> public bool GetPlayerListHtml(string url, string teamId, string teamName) { if (url.Contains("wwaattssuunn")) { return false; } HtmlParameterDTO dtomodel = new HtmlParameterDTO(); dtomodel.Url = url; dtomodel.Timeout = 10 * 1000; dtomodel.IsCheckEmpty = false; HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel);// CommonHelper.GetHtml(url, new Dictionary<string, string>(), "", "", 90000, 10); if (doc.DocumentNode.InnerHtml == "Termination") { return false; } //获取球员的列表 var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='mingd_right']/dl"); if (TongJiListData == null || TongJiListData.Count == 0) return false; foreach (var list in TongJiListData) { HtmlDocument doc1 = new HtmlDocument(); doc1.LoadHtml(list.InnerHtml); var playerLIst = doc1.DocumentNode.SelectNodes("//dd/a"); var playeerType = doc1.DocumentNode.SelectNodes("//dt"); var position = ""; if (playeerType.Count > 0) { position = playeerType.FirstOrDefault().InnerText.Replace("/t", "").Trim(); } if (playerLIst == null) { continue; } Task.Run(() => { foreach (var playitem in playerLIst) { var playerurl = playitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value; var playerName = playitem.InnerText.Trim(); var playerInfo = ""; if (allPlayer.Where(o => o.ChineseName == playerName).Count() == 0) { playerInfo = GetPlayerInfo(playerurl, playerName); } var playid = ""; var num = ""; if (playerInfo != "")//不为空表示需要新增 { playid = playerInfo.Split(',').Count() > 0 ? playerInfo.Split(',')[0] : ""; num = playerInfo.Split(',').Count() > 1 ? playerInfo.Split(',')[1] : "0"; F_Relation f_Relation = new F_Relation(); f_Relation.Id = Guid.NewGuid().ToString(); f_Relation.TeamId = teamId; f_Relation.PlayerId = playid; f_Relation.PoloShirt = num == "" ? 0 : int.Parse(num); f_Relation.Position = position; f_Relation.CreateDateTime = DateTime.Now; f_Relation.ContractTime = DateTime.Now; if (f_Relation.PlayerId != null && f_Relation.PlayerId != "") { relations.Add(f_Relation); } } } }); } return true; } /// <summary> /// 解析球员信息,并返回id /// </summary> /// <param name="url"></param> /// <param name="name"></param> /// <returns></returns> public string GetPlayerInfo(string url, string name) { var mainUrl = "http://saishi.zgzcw.com/" + url; HtmlParameterDTO dtomodel = new HtmlParameterDTO(); dtomodel.Url = url; dtomodel.Timeout = 10 * 1000; dtomodel.IsCheckEmpty = false; HtmlDocument doc = CommonHelper.GetHtmlHtmlDocument(dtomodel); //CommonHelper.GetHtml(mainUrl,new Dictionary<string, string>() ,"","",90000,10); if (doc.DocumentNode.InnerHtml == "Termination") { return ""; } //球员图片 var teamImg = doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dt/img"); //球员介绍 var teamIntroData = doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']") == null || doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']").Count == 0 ? "" : doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']").FirstOrDefault().InnerHtml.Trim(); //球员的基本信息 if (doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd") == null || doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd").Count == 0) { return ""; } var playerData = doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd").ToList(); F_Players f_Players = new F_Players(); f_Players.Id = Guid.NewGuid().ToString(); f_Players.ChineseName = name; f_Players.Describe = teamIntroData; f_Players.Remark = mainUrl; f_Players.LogoImage = teamImg.FirstOrDefault().Attributes.SingleOrDefault(a => a.Name.Equals("src")).Value; var contry = playerData.Where(o => o.InnerText.Contains("国家")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("国家")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null; f_Players.Nationality = contry != null && contry.Count() > 1 ? contry[1].Trim() : ""; var Weight = playerData.Where(o => o.InnerText.Contains("体重")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("体重")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null; f_Players.Weight = Weight != null && Weight.Count() > 1 ? Weight[1].Trim() : ""; var Stature = playerData.Where(o => o.InnerText.Contains("身高")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("身高")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null; f_Players.Stature = Stature != null && Stature.Count() > 1 ? Stature[1].Trim() : ""; var Birthday = playerData.Where(o => o.InnerText.Contains("生日")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("生日")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null; f_Players.Birthday = Birthday != null && Birthday.Count() > 1 ? Birthday[1].Trim() : ""; var num = playerData.Where(o => o.InnerText.Contains("号码")).Count() > 0 ? playerData.Where(o => o.InnerText.Contains("号码")).FirstOrDefault().InnerText.Replace("\t", "").Split(':') : null; var playerNum = num != null && num.Count() > 1 ? num[1].Trim() : ""; //if (allPlayer.Where(o => o.ChineseName == f_Players.ChineseName).Count() == 0) //{ players.Add(f_Players); return f_Players.Id + "," + playerNum; //} //services.AddPlayers(FCSLottery.F_Players, f_Players); //return ""; } #region SQL语句 private static string GetAllTeamUrl = @"select Id,Name, Remark from F_Team where Remark is not null"; #endregion } }