using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using FCS.Common; using FCS.Interface; using FCS.Models; using FCS.Models.DTO; using HtmlAgilityPack; using Quartz; using Newtonsoft.Json; using System.Text.RegularExpressions; namespace FCS.Crawler.ZCLotteryAgainst { /// /// 对阵 /// public class AgainstJob : CommonJob, IJob { private static List againstTypeList;//页面的参数 private List teamList;//比赛信息 private List eventsList;//赛事信息 private int courteenCode = 14;//14 private int sixHalfCode = 6; private int fourCode = 4; private List vreifiList;//符号验证 private List result = new List(); public AgainstJob() { teamList = services.Query().ToList(); eventsList = services.Query().ToList(); vreifiList = new List { "0",":","-" }; againstTypeList = new List { new AgainstTypeDTO{ Code=courteenCode, CaiZhong="13", Url="http://cp.zgzcw.com/lottery/zcplayvs.action", LotteryId="300", SessionUrl1="http://cp.zgzcw.com/lottery/getissue.action?issueLen=10&lotteryId=300",//"/lottery/getwqissuereturnall.action?lotteryId=300&issueLen=20&d=1538101462272" SessionUrl2="http://cp.zgzcw.com/lottery/getwqissuereturnall.action?lotteryId=300&issueLen=20&d="//"/lottery/getissue.action?lotteryId=300&issueLen=20&d=1538101378587" }, new AgainstTypeDTO{ Code=sixHalfCode, CaiZhong="15", Url="http://cp.zgzcw.com/lottery/zcplayvs.action", LotteryId="302", SessionUrl1="http://cp.zgzcw.com/lottery/getissue.action?issueLen=10&lotteryId=302", SessionUrl2="http://cp.zgzcw.com/lottery/getwqissuereturnall.action?lotteryId=302&issueLen=20&d=" }, new AgainstTypeDTO{ Code=fourCode, CaiZhong="16", Url="http://cp.zgzcw.com/lottery/zcplayvs.action", LotteryId="303", SessionUrl1="http://cp.zgzcw.com/lottery/getissue.action?issueLen=10&lotteryId=303", SessionUrl2="http://cp.zgzcw.com/lottery/getwqissuereturnall.action?lotteryId=303&issueLen=20&d=" }, }; } public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); Click(); } public void Click() { againstTypeList.ForEach(d => { var sessionList = new List(); if (d.Code == courteenCode) sessionList = GetSession(d, true).ToList(); else if (d.Code == sixHalfCode) sessionList = GetSession(d, true).ToList(); else sessionList = GetSession(d, true).ToList(); threadCount = sessionList.Count; finishcount = 0; sessionList.ForEach(p => { Analysis(d, p); }); lock (locker) { while (finishcount != threadCount) { Thread.Sleep(5000); Monitor.Wait(locker);//等待 } } var deleteWhere = ""; sessionList.ForEach(p => { deleteWhere += p + ","; }); deleteWhere = deleteWhere.ToString().Substring(0, deleteWhere.ToString().Length - 1); if (d.Code == courteenCode) { services.Delete(" AND Session in ({0})".FormatMe(deleteWhere)); services.SqlBulkCopyAdd(Mapper>(result)); } else if (d.Code == sixHalfCode) { services.Delete(" AND Session in ({0})".FormatMe(deleteWhere)); services.SqlBulkCopyAdd(Mapper>(result)); } else { services.Delete(" AND Session in ({0})".FormatMe(deleteWhere)); services.SqlBulkCopyAdd(Mapper>(result)); } }); } public void GetALL() { againstTypeList.ForEach(d => { var sessionList = GetSession(d).ToList(); var result = new List(); sessionList.ForEach(p => { Analysis(d, p); }); while (true) { if (CommonHelper.ThreadsFinsh()) break; } if (d.Code == courteenCode) services.SqlBulkCopyAdd(Mapper>(result)); else if (d.Code == sixHalfCode) services.SqlBulkCopyAdd(Mapper>(result)); else services.SqlBulkCopyAdd(Mapper>(result)); }); } private async Task Analysis(AgainstTypeDTO d, string p) { await Task.Run(() => { try { //http://cp.zgzcw.com/lottery/zcplayvs.action?lotteryId=13&issue=18094&v=1533868879113 var url = d.Url + "?lotteryId=" + d.CaiZhong + "&issue=" + p + "&v=" + CommonHelper.ConvertDateTimeToInt(DateTime.Now); var doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url }); while (doc.DocumentNode.InnerText.ToLower().Contains("html"))//如果包含Html,不符合 doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url }); var data = JsonConvert.DeserializeObject(doc.DocumentNode.InnerText); data.matchInfo.ToList().ForEach(q => { var index = data.matchInfo.ToList().IndexOf(q); result.Add(new AgainstEntity { Id = CommonHelper.GetGuid().ToString(), Session = int.Parse(q.issue), StartDateTime = DateTime.Parse(q.gameStartDate), HomeTeamId = (from a in teamList where a.Name == Regex.Replace(q.hostName, @"\s", "") select a.Id).Take(1).ToJoin(), VisitingTeamId = (from a in teamList where a.Name == Regex.Replace(q.guestName, @"\s", "") select a.Id).Take(1).ToJoin(), HomeTeamName = q.hostName, VisitingTeamName = q.guestName, HalfResult = vreifiList.Contains(q.zuizhongbifen.Trim()) ? string.Empty : d.Code == sixHalfCode ? q.zuizhongbifen.Split(';')[index].Split(',')[0] : string.Empty, Result = vreifiList.Contains(q.zuizhongbifen.Trim()) ? string.Empty : d.Code == sixHalfCode ? q.zuizhongbifen.Split(';')[index].Split(',')[1] : q.zuizhongbifen.Split(';')[index], HalfColourFruit = d.Code == sixHalfCode || d.Code == fourCode ? q.code == "0" ? string.Empty : q.code.Split(',')[index * 2 + 1] : string.Empty, ColourFruit = d.Code == sixHalfCode || d.Code == fourCode ? q.code == "0" ? string.Empty : q.code.Split(',')[index * 2 + 2] : q.code == "0" ? string.Empty : q.code.Split(',')[index], Compensate_SOdd = q.europeSp.Trim().IsEmpty() ? 0 : float.Parse(q.europeSp.Split(' ')[0]), Compensate_POdd = q.europeSp.Trim().IsEmpty() ? 0 : float.Parse(q.europeSp.Split(' ')[1]), Compensate_FOdd = q.europeSp.Trim().IsEmpty() ? 0 : float.Parse(q.europeSp.Split(' ')[2]), AsianDish_SOdd = q.yapan.Trim().IsEmpty() ? 0 : float.Parse(q.europeSp.Split(' ')[0]), AsianDish_Disc = q.yapan.Trim().IsEmpty() ? "" : q.yapan.Split(' ')[1], AsianDish_FOdd = q.yapan.Trim().IsEmpty() ? 0 : float.Parse(q.yapan.Split(' ')[2]), EventId = eventsList.Where(s => s.Name.Contains(Regex.Replace(q.leageName, @"\s", ""))).ToList()[0].Id, CreateDateTime = DateTime.Now, Sort = index + 1 }); }); } catch (Exception) { } finally { lock (locker) { finishcount++; Monitor.Pulse(locker); //完成,通知等待队列,告知已完,执行下一个。 } } }); } /// /// 得到期数 /// /// /// private IEnumerable GetSession(AgainstTypeDTO model, bool isUpdate = false) { var sessionList = new List(); if (isUpdate) { var nosessionList = services.Query(" AND Result IS NULL OR Result =''", "", "a.Session").ToList(); if (nosessionList.Count <= 0) { nosessionList = services.Query("", "", "max(a.Session) as Session").ToList(); if (nosessionList.Count > 0) sessionList = (from a in nosessionList select a.Session.ToString()).Distinct().ToList(); } else { sessionList = (from a in nosessionList select a.Session.ToString()).Distinct().ToList(); } } //期数1 var doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = model.SessionUrl1 }); while (doc.DocumentNode.InnerText.ToLower().Contains("html")) doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = model.SessionUrl1 }); var list = JsonConvert.DeserializeObject>(doc.DocumentNode.InnerText); //期数2 doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = model.SessionUrl2 + CommonHelper.ConvertDateTimeToInt(DateTime.Now) }); while (doc.DocumentNode.InnerText.ToLower().Contains("html")) doc = CommonHelper.GetHtmlHtmlDocument(new HtmlParameterDTO { Url = model.SessionUrl2 + CommonHelper.ConvertDateTimeToInt(DateTime.Now) }); var list1 = JsonConvert.DeserializeObject>(doc.DocumentNode.InnerText); //判断依据来自:zc.min.js(对于页面的js分析得出) list1 = (from a in list1 where a.status != "9" && !(from b in list select b.issue).Contains(a.issue) select a).ToList(); list = list.Concat(list1).Distinct().ToList(); if (sessionList != null) { if (sessionList.Count > 0) { var list_y = from a in list join b in sessionList on a.issue equals b select b; if (list_y.Count() <= 0) list_y = sessionList; var session = list_y.OrderBy(p => p).ToList()[0]; return (from a in list where int.Parse(a.issue) >= int.Parse(session.IsEmpty() ? "0" : session) select a.issue ).Distinct(); } } return (from a in list select a.issue ).Distinct(); } } }