using System; using System.Collections.Generic; using System.Linq; using HtmlAgilityPack; using Quartz; using SCC.Common; using SCC.Crawler.Tools; using SCC.Interface; using SCC.Models; using SCC.Services; namespace SCC.Crawler.GP { /// /// 北京PK10(北京赛车) /// [DisallowConcurrentExecution] [PersistJobDataAfterExecution] public class BJPK10Job : IJob { /// /// 构造函数 /// public BJPK10Job() { log = new LogHelper(); services = IOC.Resolve(); email = IOC.Resolve(); } /// /// 作业执行入口 /// /// 作业执行上下文 public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); //预设节假日不开奖 if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return; LatestItem = context.JobDetail.JobDataMap["LatestItem"] as OpenCode10Model; try { //服务启动时配置初始数据 if (LatestItem == null) LatestItem = services.GetLastItem(currentLottery); //第一次启动服务或最新开奖记录为昨日滴,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表 if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd")) { CheckingYesterdayTheLotteryData(); LatestItem = GenerateTodayFirstItem(); } //当最新开奖记录不是今天最后一期,执行当天作业 if (!CheckLatestItemIsTodayLastItem()) { DoTodayJobByMainUrl(); DoTodayJobByBackUrl(); } } catch (Exception ex) { log.Error(GetType(), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + currentLottery, ex.Message)); } //保存最新期号和失败期号列表 context.JobDetail.JobDataMap["LatestItem"] = LatestItem; } /// /// 自检昨天开奖数据 /// private void CheckingYesterdayTheLotteryData() { if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return; //如果昨日设定不开奖则不自检昨日开奖数据 //从数据库中获取昨天数据抓取失败列表 FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, GenerateYesterdayQiHaoList()); if (FailedQiHaoList.Count > 0) { DoYesterdayFailedListByMainUrl(); DoYesterdayFailedListByBackUrl(); } } /// /// 通过主站点抓取开奖数据 /// (百度彩票) /// private void DoTodayJobByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl)) { var OpenList = GetDocByMainUrl(); if (OpenList.Count == 0) return; //无抓取数据 var startQiNum = LatestItem.Term + 1; var newestQiNum = OpenList.Max(w => w.Term); if (startQiNum > newestQiNum) return; //无最新数据 var total = OpenList.Count; //处理最新开奖数据 for (var i = startQiNum; i <= newestQiNum; i++) { var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault(); var step = 0; var nowQiHao = Convert.ToInt32(i); while (matchItem == null) if (step <= total) { nowQiHao++; step++; matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault(); } else { matchItem = null; break; } if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(GetType(), CommonHelper.GetJobMainLogInfo(Config, i.ToString())); LatestItem = matchItem; } } } } /// /// 通过主站抓取错误期号列表中每一个期号 /// (百度彩票) /// private void DoYesterdayFailedListByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetDocByMainUrl(false); if (OpenList.Count == 0) return; //无抓取数据 var total = OpenList.Count; var SuccessList = new List(); foreach (var failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault(); var step = 0; var nowQiHao = Convert.ToInt32(failedQiHao); while (matchItem == null) if (step <= total) { nowQiHao++; step++; matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault(); } else { matchItem = null; break; } if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(GetType(), CommonHelper.GetJobMainLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); } } foreach (var successQiHao in SuccessList) FailedQiHaoList.Remove(successQiHao); } } /// 从主站抓取开奖数据 昨天/今天 /// /// /// private List GetDocByMainUrl(bool IsToday = true) { var list = new List(); try { var time = DateTime.Now; var arg = new OpenCaiApiArg { code = EnumHelper.GetLotteryCode(SCCLottery.BeiJingPK10), date = time.ToString("yyyy-MM-dd") }; if (!IsToday) arg.date = time.AddDays(-1).ToString("yyyy-MM-dd"); var data = OpenCaiApiServices.GetOpenCaiApiData(arg); if (data == null) return list; if (data.data != null) { if (data.data.Count == 0) return list; for (var i = 1; i <= data.data.Count; i++) { var tmp = new OpenCode10Model(); var codeAry = data.data[i - 1].GetOpenCodeStr().Split(','); if (codeAry.Length == 10) { tmp.Term = long.Parse(data.data[i - 1].expect); tmp.OpenTime = DateTime.Parse(data.data[i - 1].opentime); tmp.OpenCode1 = int.Parse(codeAry[0]); tmp.OpenCode2 = int.Parse(codeAry[1]); tmp.OpenCode3 = int.Parse(codeAry[2]); tmp.OpenCode4 = int.Parse(codeAry[3]); tmp.OpenCode5 = int.Parse(codeAry[4]); tmp.OpenCode6 = int.Parse(codeAry[5]); tmp.OpenCode7 = int.Parse(codeAry[6]); tmp.OpenCode8 = int.Parse(codeAry[7]); tmp.OpenCode9 = int.Parse(codeAry[8]); tmp.OpenCode10 = int.Parse(codeAry[9]); } list.Add(tmp); } var checkDataHelper = new CheckDataHelper(); var dbdata = services.GetListIn(currentLottery, IsToday); if (dbdata == null) return list; var DATA = dbdata.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()); checkDataHelper.CheckData(DATA, list.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()), Config.Area, currentLottery); } } catch (Exception ee) { log.Error(GetType(), string.Format("【{0}】通过站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + currentLottery, ee.Message)); } return list; } /// /// 通过备用站点抓取开奖数据 /// (北京福彩官网) /// private void DoTodayJobByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl)) { var OpenList = GetOpenListFromBackUrl(Config.BackUrl); if (OpenList.Count == 0) return; //无抓取数据 var startQiNum = LatestItem.Term + 1; var newestQiNum = OpenList.Max(w => w.Term); if (startQiNum > newestQiNum) return; //无最新数据 var total = OpenList.Count; //处理最新开奖数据 for (var i = startQiNum; i <= newestQiNum; i++) { var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault(); var step = 0; var nowQiHao = Convert.ToInt32(i); while (matchItem == null) if (step <= total) { nowQiHao++; step++; matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault(); } else { matchItem = null; break; } if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(GetType(), CommonHelper.GetJobBackLogInfo(Config, i.ToString())); LatestItem = matchItem; } } } } /// /// 通过备用地址抓取错误期号列表中每一个期号 /// (北京福彩官网) /// private void DoYesterdayFailedListByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetOpenListFromBackUrl(Config.BackUrl, false); if (OpenList.Count == 0) return; //无抓取数据 var total = OpenList.Count; var SuccessList = new List(); foreach (var failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault(); var step = 0; var nowQiHao = Convert.ToInt32(failedQiHao); while (matchItem == null) if (step <= total) { nowQiHao++; step++; matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault(); } else { matchItem = null; break; } if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(GetType(), CommonHelper.GetJobBackLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); } } foreach (var successQiHao in SuccessList) FailedQiHaoList.Remove(successQiHao); } } /// /// 抓取备用站点开奖数据 /// 北京福彩官网分页数据太多,只有通过主站点来保证开奖数据完整性 /// /// 备用站点 /// private List GetOpenListFromBackUrl(string url, bool IsToday = true) { var result = new List(); try { var HtmlResource = string.Empty; HtmlResource = NetHelper.GetUrlResponse(url); if (HtmlResource == null) return result; var doc = new HtmlDocument(); doc.LoadHtml(HtmlResource); var table = doc.DocumentNode.SelectNodes("//table"); if (table == null || table.Count < 2) return result; var trs = table[1].ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList(); if (trs.Count < 2) return result; List tds = null; OpenCode10Model model = null; string[] openCodeList = null; var openTime = string.Empty; var todayDateString = CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd"); for (var i = 1; i < trs.Count; i++) //第一行是表头 { tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds == null) return result; if (tds.Count < 3) continue; openTime = tds[2].InnerText.Trim(); if (!openTime.StartsWith(todayDateString)) continue; model = new OpenCode10Model(); model.Term = Convert.ToInt64(tds[0].InnerText.Trim()); openCodeList = tds[1].InnerText.Trim().Split(','); if (openCodeList.Length < 10) continue; model.OpenCode1 = Convert.ToInt32(openCodeList[0]); model.OpenCode2 = Convert.ToInt32(openCodeList[1]); model.OpenCode3 = Convert.ToInt32(openCodeList[2]); model.OpenCode4 = Convert.ToInt32(openCodeList[3]); model.OpenCode5 = Convert.ToInt32(openCodeList[4]); model.OpenCode6 = Convert.ToInt32(openCodeList[5]); model.OpenCode7 = Convert.ToInt32(openCodeList[6]); model.OpenCode8 = Convert.ToInt32(openCodeList[7]); model.OpenCode9 = Convert.ToInt32(openCodeList[8]); model.OpenCode10 = Convert.ToInt32(openCodeList[9]); model.OpenTime = Convert.ToDateTime(openTime); if (!result.Contains(model)) result.Add(model); } var checkDataHelper = new CheckDataHelper(); var dbdata = services.GetListIn(currentLottery, IsToday); if (dbdata == null) return result; var datas = dbdata.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()); checkDataHelper.CheckData(datas, result.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()), Config.Area, currentLottery); } catch (Exception ex) { log.Error(GetType(), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + currentLottery, ex.Message)); } return result; } /// /// 生成今日第一期记录 /// /// private OpenCode10Model GenerateTodayFirstItem() { var item = new OpenCode10Model(); var currentDateTime = CommonHelper.SCCSysDateTime; var datepart = currentDateTime - new DateTime(2018, 1, 1); var t = Config.SkipDate.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries); item.Term = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear; //期号使用昨日最后一期期号,便于计算使用 item.OpenTime = new DateTime(currentDateTime.Year, currentDateTime.Month, currentDateTime.Day, Config.StartHour, Config.StartMinute, 0); return item; } /// /// 核实最新一期开奖记录是否为今日最后一期 /// /// 期号 /// private bool CheckLatestItemIsTodayLastItem() { if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd")) return true; var firstItem = GenerateTodayFirstItem(); if (LatestItem.Term == firstItem.Term + Config.TimesPerDay) return true; return false; } /// /// 获取今天第一期期号 /// /// private string GenerateTodayFirstQiHao() { var datepart = CommonHelper.SCCSysDateTime - new DateTime(2018, 1, 1); var t = Config.SkipDate.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries); var terms = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear + 1; return terms.ToString(); } /// /// 生成昨天的期号列表 /// /// private List GenerateYesterdayQiHaoList() { var result = new List(); var todayFirstQiHao = Convert.ToInt64(GenerateTodayFirstQiHao()); for (var i = Config.TimesPerDay; i > 0; i--) result.Add((todayFirstQiHao - i).ToString()); return result; } /// /// 生成PK10的开奖时间 /// /// /// private DateTime GenerateYesterdayPK10OpenTime(string QiHao) { var openDay = CommonHelper.SCCSysDateTime.AddDays(-1); var StartTime = new DateTime(openDay.Year, openDay.Month, openDay.Day, Config.StartHour, Config.StartMinute, 0); var t = (Convert.ToInt64(QiHao) - LastTermLastYear) % Config.TimesPerDay; if (t == 0) return StartTime.AddMinutes((Config.TimesPerDay - 1) * Config.Interval); return StartTime.AddMinutes((t - 1) * Config.Interval); } #region Attribute /// /// 配置信息 /// private SCCConfig Config; /// /// 当天抓取的最新一期开奖记录 /// private OpenCode10Model LatestItem; /// /// 当天抓取失败列表 /// private List FailedQiHaoList; /// /// 日志对象 /// private readonly LogHelper log; /// /// 数据服务 /// private readonly IOpen10Code services; /// /// 当前彩种 /// private SCCLottery currentLottery => SCCLottery.BeiJingPK10; /// /// 邮件接口 /// private IEmail email; /// /// 2016年最后一期期号 /// private readonly long LastTermLastYear = 659223; #endregion } }