using System; using System.Collections.Generic; using System.Configuration; using System.Diagnostics; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using FCS.Common; using FCS.Interface; using FCS.Models; using FCS.Models.DTO; using HtmlAgilityPack; using Quartz; namespace FCS.Crawler.ZCLotteryScore {/// /// 创 建:czx /// 日 期:2018-09-27 /// 描 述:比分数据更新、获取 /// public class ScoreJob : CommonJob { #region 全局变量定义 private List scoreTypeList;//类别枚举类型 private List scoreStatusList;//状态枚举类型 private Dictionary scoreStatusDict;//状态枚举类型-字典 private List scoreJobList;//页面爬取参数 private List eventsList;//赛事信息 private List teamList;//球队信息 private List gameList;//球队信息 private string starting = "2";//开始中 private string sfcCode = "300";//足彩 private List result = new List(); #endregion public ScoreJob() { scoreTypeList = services.GetDataItem(DataItemDetailEnum.FootScoreType); scoreStatusList = services.GetDataItem(DataItemDetailEnum.FootScoreStatus); eventsList = services.Query().ToList(); teamList = services.Query().ToList(); gameList = services.Query(" AND Datename(year,StartDateTime)={0}".FormatMe(DateTime.Now.Year)).ToList(); scoreStatusDict = new Dictionary { {"未","1" }, {"完","3" }, {"推迟 ","4" } }; scoreJobList = new List{ new ScoreJobDTO{ Type="2", Code="201", SessionName="date", Ajax="true", Url="http://live.zgzcw.com/jz/" },//竞彩 new ScoreJobDTO{ Type="3", Code="300", SessionName ="issue", Ajax="true", Url="http://live.zgzcw.com/sfc/" },//足彩 new ScoreJobDTO{ Type="4", Code="400", SessionName ="issue", Ajax="true", Url="http://live.zgzcw.com/bd/" },//北单 }; } /// /// 更新 /// public void Click() { scoreJobList.ForEach(p => { result = new List(); var scoreType = scoreTypeList.Where(s => s.ItemValue == p.Type).ToList()[0].Id; //期数数据 var sessionList = GetSession(p.Url, scoreType, true).ToList(); threadCount = sessionList.Count; finishcount = 0; sessionList.ForEach(q => { Analysis(p, q, scoreType); }); lock (locker) { while (finishcount != threadCount) { Thread.Sleep(5000); Monitor.Wait(locker);//等待 } } var deleteWhere = ""; sessionList.ForEach(d => { deleteWhere += d.Replace("-", "") + ","; }); deleteWhere = deleteWhere.ToString().Substring(0, deleteWhere.ToString().Length - 1); services.Delete("AND ScoreType='{0}' AND Session in ({1})".FormatMe(scoreType, deleteWhere)); services.SqlBulkCopyAdd(result); }); } /// /// 获取全部 /// /// public void GetALL(string url = "http://live.zgzcw.com/ls/AllData.action") { var result = new List(); scoreJobList.ForEach(p => { Task.Run(() => { //期数数据 var sessionList = GetSession(p.Url).ToList(); sessionList.ForEach(q => { Analysis(p, q); }); }); }); while (true) { if (CommonHelper.ThreadsFinsh()) break; } services.SqlBulkCopyAdd(result); } /// /// 数据分析 /// /// 比分 /// 期数 /// 比分类型 /// private async Task Analysis(ScoreJobDTO p, string q, string type = "") { string url = "http://live.zgzcw.com/ls/AllData.action"; await Task.Run(() => { try { var dict = new Dictionary {{ "code",p.Code},{ p.SessionName,q},{ "ajax",p.Ajax}};//formData参数拼接 var doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url, FormData = dict, Title = "matchid", Method = "POST" }); var tr = doc.DocumentNode.SelectNodes(".//tr"); while (tr == null && doc.DocumentNode.InnerHtml != ConfigurationManager.AppSettings["Termination"]) { doc = doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url, FormData = dict, Title = "matchid", Method = "POST" }); tr = doc.DocumentNode.SelectNodes(".//tr"); } if (doc.DocumentNode.InnerHtml == ConfigurationManager.AppSettings["Termination"])//不存在退出 return; foreach (HtmlNode item in tr) { var td = item.SelectNodes(".//td"); if (td == null) continue; if (td.Count < 5) break; string _result = "", status = "", halfResultatus = "", colourFruit = "", gameId = ""; if (scoreStatusDict.ContainsKey(td[4].InnerText.Trim())) { status = scoreStatusList.Where(a => a.ItemValue.Contains(scoreStatusDict[td[4].InnerText.Trim()])).ToList()[0].Id; _result = td[6].SelectNodes(".//span")[0].InnerHtml.Replace(" ", ""); halfResultatus = td[8].SelectNodes(".//span")[0].InnerHtml.Replace(" ", ""); } else status = scoreStatusList.Where(a => a.ItemValue.Contains(starting)).ToList()[0].Id; if (p.Code == sfcCode) colourFruit = td[9].SelectNodes(".//strong") != null ? td[9].SelectNodes(".//strong")[0].InnerText : string.Empty; var compensate = tr[tr.IndexOf(item)].SelectSingleNode(".//div[@class='oupei']").SelectNodes("span"); var asianDish = tr[tr.IndexOf(item)].SelectSingleNode(".//div[@class='yapan']").SelectNodes("span"); var model = new F_Score { Id = CommonHelper.GetGuid().ToString(), EventId = eventsList.Where(s => s.Name.Contains(td[1].InnerText)).ToList().Count > 0 ? eventsList.Where(s => s.Name.Contains(td[1].InnerText)).ToList()[0].Id : "", GroupName = td[2].InnerText, StartDateTime = DateTime.Parse(td[3].Attributes["date"].Value), HomeTeamId = (from a in teamList where a.Name == Regex.Replace(td[5].SelectNodes(".//a")[0].InnerText, @"\s", "") select a.Id).Take(1).ToJoin(), VisitingTeamId = (from a in teamList where a.Name == Regex.Replace(td[7].SelectNodes(".//a")[0].InnerText, @"\s", "") select a.Id).Take(1).ToJoin(), HomeTeamName = td[5].SelectNodes(".//a")[0].InnerText, VisitingTeamName = td[7].SelectNodes(".//a")[0].InnerText, Status = status, EventName = td[1].SelectNodes(".//span")[0].InnerText, Result = _result.Trim() == "-" ? string.Empty : _result.Trim(), HalfResult = halfResultatus.Trim() == "-" ? string.Empty : halfResultatus.Trim(), ColourFruit = colourFruit, Session = int.Parse(q.Replace("-", "")), Compensate_SOdd = float.Parse(compensate == null ? "0" : compensate[0].InnerText), Compensate_POdd = float.Parse(compensate == null ? "0" : compensate[1].InnerText), Compensate_FOdd = float.Parse(compensate == null ? "0" : compensate[2].InnerText), AsianDish_SOdd = float.Parse(asianDish == null ? "0" : asianDish[0].InnerText), AsianDish_Disc = asianDish != null ? asianDish[1].InnerText : string.Empty, AsianDish_FOdd = float.Parse(asianDish == null ? "0" : asianDish[2].InnerText), CreateDateTime = DateTime.Now, ScoreType = type.IsEmpty() ? scoreTypeList.Where(s => s.ItemValue == p.Type).ToList()[0].Id : type, Sort = tr.IndexOf(item) + 1 }; model.GameId = (from a in gameList where a.HomeTeamId == model.HomeTeamId && a.VisitingTeamId == model.VisitingTeamId && ((DateTime)a.StartDateTime).ToString("yyyy-MM-dd") == model.StartDateTime.ToString("yyyy-MM-dd") select a.Id).Take(1).ToJoin(); result.Add(model); } } catch (Exception ex) { } finally { lock (locker) { finishcount++; Monitor.Pulse(locker); //完成,通知等待队列,告知已完,执行下一个。 } } }); } /// /// 获取期数 /// /// /// /// /// private IEnumerable GetSession(string url, string type = "", bool isUpdate = false) { var list = new List(); var sessionList = new List(); if (isUpdate) { var nosessionList = services.Query(" AND (Result IS NULL OR Result ='') AND ScoreType='{0}'".FormatMe(type), "", "a.Session").ToList(); if (nosessionList.Count <= 0) { nosessionList = services.Query(" AND ScoreType='{0}'".FormatMe(type), "", "max(a.Session) as Session").ToList(); if (nosessionList.Count > 0) sessionList = (from a in nosessionList select a.Session.ToString()).Distinct().ToList(); } else { sessionList = (from a in nosessionList select a.Session.ToString()).Distinct().ToList(); } } var doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url, Title = "足彩" ,Timeout=10000}); var opent = doc.DocumentNode.SelectSingleNode(".//select[@id='matchSel']").SelectNodes(".//option"); foreach (var item in opent) { list.Add(item.Attributes["value"].Value); } if (sessionList.Count > 0) { var list_y = from a in list join b in sessionList on a.Replace("-", "") equals b select b; if (list_y.Count() <= 0) list_y = sessionList; var session = list_y.OrderBy(p => p).ToList()[0]; return (from a in list where int.Parse(a.Replace("-", "")) >= int.Parse(session) select a ).Distinct().ToList(); } return list; } } }