using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Xml; using Quartz; using SCC.Common; using SCC.Models; using SCC.Interface; using HtmlAgilityPack; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace SCC.Crawler.GP { /// /// 数据爬取类 /// 北京快乐8 /// [DisallowConcurrentExecution] [PersistJobDataAfterExecution] public class BJKL8Job : IJob { /// /// 构造函数 /// public BJKL8Job() { log = new LogHelper(); services = IOC.Resolve(); email = IOC.Resolve(); } /// /// 作业执行入口 /// /// 作业执行上下文 public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); //预设节假日不开奖 if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return; LatestItem = context.JobDetail.JobDataMap["LatestItem"] as OpenCode21Model; try { //服务启动时配置初始数据 if (LatestItem == null) { LatestItem = services.GetLastItem(currentLottery); } //第一次启动服务或最新开奖记录为昨日滴,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表 if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd")) { CheckingYesterdayTheLotteryData(); LatestItem = GenerateTodayFirstItem(); } //当最新开奖记录不是今天最后一期,执行当天作业 if (!CheckLatestItemIsTodayLastItem()) { DoTodayJobByMainUrl(); //DoTodayJobByBackUrl(); } } catch (Exception ex) { log.Error(typeof(BJKL8Job), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } //保存最新期号和失败期号列表 context.JobDetail.JobDataMap["LatestItem"] = LatestItem; } /// /// 自检昨天开奖数据 /// private void CheckingYesterdayTheLotteryData() { if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据 //从数据库中获取昨天数据抓取失败列表 FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, GenerateYesterdayQiHaoList()); if (FailedQiHaoList.Count > 0) { GetYesterdayOpenListFromMainUrl_New(); //DoYesterdayFailedListByMainUrl(); //DoYesterdayFailedListByBackUrl(); foreach (var fQiHao in FailedQiHaoList) { //将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中 if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, GenerateYesterdayKL8OpenTime(fQiHao))) log.Error(typeof(BJKL8Job), CommonHelper.GetJobLogError(Config, fQiHao)); } } } /// /// 通过主站点抓取开奖数据 /// (百度彩票) /// private void DoTodayJobByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl)) { var OpenList = GetTodayOpenListFromMainUrl(); if (OpenList.Count == 0) return;//无抓取数据 var startQiNum = LatestItem.Term; var newestQiNum = OpenList.First().Term; if (startQiNum > newestQiNum) return;//无最新数据 //处理最新开奖数据 for (var i = startQiNum; i <= newestQiNum; i++) { var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault(); if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(BJKL8Job), CommonHelper.GetJobMainLogInfo(Config, i.ToString())); LatestItem = matchItem; } } } } /// /// 通过主站抓取错误期号列表中每一个期号 /// (百度彩票) /// private void DoYesterdayFailedListByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetYesterdayOpenListFromMainUrl(); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (string failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault(); if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(BJKL8Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } /// /// 通过主站点抓取今日最新开奖数据 /// /// private List GetTodayOpenListFromMainUrl() { List result = new List(); try { var requestUrl = string.Format(Config.MainUrl, DateTime.Now.ToString("yyyy-MM-dd")); var HtmlResource = NetHelper.GetUrlResponse(requestUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { OpenCode21Model model = null; JObject obj = JsonConvert.DeserializeObject(HtmlResource); var data = obj["data"]; var list = JsonConvert.DeserializeObject>(data.ToString()); foreach (var item in list) { var red = item.opencode.Split(','); var extra = red[19].Split('+'); red[19] = extra[0]; model = new OpenCode21Model(); model.Term = Convert.ToInt64(item.expect);//开奖期号 model.OpenTime = Convert.ToDateTime(item.opentime);//开奖时间 model.OpenCode21 = Convert.ToInt32(extra[1].ToString());//飞盘 model.OpenCode1 = Convert.ToInt32(red[0].ToString()); model.OpenCode2 = Convert.ToInt32(red[1].ToString()); model.OpenCode3 = Convert.ToInt32(red[2].ToString()); model.OpenCode4 = Convert.ToInt32(red[3].ToString()); model.OpenCode5 = Convert.ToInt32(red[4].ToString()); model.OpenCode6 = Convert.ToInt32(red[5].ToString()); model.OpenCode7 = Convert.ToInt32(red[6].ToString()); model.OpenCode8 = Convert.ToInt32(red[7].ToString()); model.OpenCode9 = Convert.ToInt32(red[8].ToString()); model.OpenCode10 = Convert.ToInt32(red[9].ToString()); model.OpenCode11 = Convert.ToInt32(red[10].ToString()); model.OpenCode12 = Convert.ToInt32(red[11].ToString()); model.OpenCode13 = Convert.ToInt32(red[12].ToString()); model.OpenCode14 = Convert.ToInt32(red[13].ToString()); model.OpenCode15 = Convert.ToInt32(red[14].ToString()); model.OpenCode16 = Convert.ToInt32(red[15].ToString()); model.OpenCode17 = Convert.ToInt32(red[16].ToString()); model.OpenCode18 = Convert.ToInt32(red[17].ToString()); model.OpenCode19 = Convert.ToInt32(red[18].ToString()); model.OpenCode20 = Convert.ToInt32(red[19].ToString()); if (!result.Contains(model)) result.Add(model); } } } catch (Exception ex) { log.Error(typeof(BJKL8Job), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过主站点抓取昨日开奖数据 /// /// private List GetYesterdayOpenListFromMainUrl() { List result = new List(); try { var requestUrl = string.Format("{0}/{1}", Config.MainUrl, CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd")); var HtmlResource = NetHelper.GetBaiDuLeCaiResponse(requestUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { Regex reg = new Regex(@"var phaseData = ([\s\S]*?);"); Match m = reg.Match(HtmlResource); if (!m.Success) return result; var dataJson = m.Result("$1"); JObject obj = JsonConvert.DeserializeObject(dataJson); JToken items = obj[CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd")]; if (items == null || !items.HasValues) return result; OpenCode21Model model = null; JObject childObject = null, openCode = null; JArray extra = null, red = null; foreach (JProperty item in items) { model = new OpenCode21Model(); model.Term = Convert.ToInt64(item.Name);//开奖期号 childObject = (JObject)item.Value; model.OpenTime = Convert.ToDateTime(childObject["open_time"]);//开奖时间 openCode = (JObject)childObject["result"]; extra = (JArray)openCode["extra"]; if (extra.Count != 1) continue; model.OpenCode21 = Convert.ToInt32(extra[0].ToString());//飞盘 red = (JArray)openCode["red"];//20个开奖球号 if (red.Count != 20) continue; model.OpenCode1 = Convert.ToInt32(red[0].ToString()); model.OpenCode2 = Convert.ToInt32(red[1].ToString()); model.OpenCode3 = Convert.ToInt32(red[2].ToString()); model.OpenCode4 = Convert.ToInt32(red[3].ToString()); model.OpenCode5 = Convert.ToInt32(red[4].ToString()); model.OpenCode6 = Convert.ToInt32(red[5].ToString()); model.OpenCode7 = Convert.ToInt32(red[6].ToString()); model.OpenCode8 = Convert.ToInt32(red[7].ToString()); model.OpenCode9 = Convert.ToInt32(red[8].ToString()); model.OpenCode10 = Convert.ToInt32(red[9].ToString()); model.OpenCode11 = Convert.ToInt32(red[10].ToString()); model.OpenCode12 = Convert.ToInt32(red[11].ToString()); model.OpenCode13 = Convert.ToInt32(red[12].ToString()); model.OpenCode14 = Convert.ToInt32(red[13].ToString()); model.OpenCode15 = Convert.ToInt32(red[14].ToString()); model.OpenCode16 = Convert.ToInt32(red[15].ToString()); model.OpenCode17 = Convert.ToInt32(red[16].ToString()); model.OpenCode18 = Convert.ToInt32(red[17].ToString()); model.OpenCode19 = Convert.ToInt32(red[18].ToString()); model.OpenCode20 = Convert.ToInt32(red[19].ToString()); if (!result.Contains(model)) result.Add(model); } } } catch (Exception ex) { log.Error(typeof(BJKL8Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过主站点抓取昨日开奖数据 /// /// private List GetYesterdayOpenListFromMainUrl_New() { List result = new List(); try { var requestUrl = string.Format(Config.MainUrl,DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd")); var HtmlResource = NetHelper.GetUrlResponse(requestUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { OpenCode21Model model = null; JObject obj = JsonConvert.DeserializeObject(HtmlResource); var data = obj["data"]; var SuccessList = new List(); var list = JsonConvert.DeserializeObject>(data.ToString()); foreach (var item in list) { var red = item.opencode.Split(','); var extra = red[19].Split('+'); red[19] = extra[0]; model = new OpenCode21Model(); model.Term = Convert.ToInt64(item.expect);//开奖期号 model.OpenTime = Convert.ToDateTime(item.opentime);//开奖时间 model.OpenCode21 = Convert.ToInt32(extra[1].ToString());//飞盘 model.OpenCode1 = Convert.ToInt32(red[0].ToString()); model.OpenCode2 = Convert.ToInt32(red[1].ToString()); model.OpenCode3 = Convert.ToInt32(red[2].ToString()); model.OpenCode4 = Convert.ToInt32(red[3].ToString()); model.OpenCode5 = Convert.ToInt32(red[4].ToString()); model.OpenCode6 = Convert.ToInt32(red[5].ToString()); model.OpenCode7 = Convert.ToInt32(red[6].ToString()); model.OpenCode8 = Convert.ToInt32(red[7].ToString()); model.OpenCode9 = Convert.ToInt32(red[8].ToString()); model.OpenCode10 = Convert.ToInt32(red[9].ToString()); model.OpenCode11 = Convert.ToInt32(red[10].ToString()); model.OpenCode12 = Convert.ToInt32(red[11].ToString()); model.OpenCode13 = Convert.ToInt32(red[12].ToString()); model.OpenCode14 = Convert.ToInt32(red[13].ToString()); model.OpenCode15 = Convert.ToInt32(red[14].ToString()); model.OpenCode16 = Convert.ToInt32(red[15].ToString()); model.OpenCode17 = Convert.ToInt32(red[16].ToString()); model.OpenCode18 = Convert.ToInt32(red[17].ToString()); model.OpenCode19 = Convert.ToInt32(red[18].ToString()); model.OpenCode20 = Convert.ToInt32(red[19].ToString()); if (item.expect != null && services.AddOpen21Code(currentLottery, model)) { //处理成功写入日志 log.Info(typeof(BJKL8Job), CommonHelper.GetJobMainLogInfo(Config, item.expect)); SuccessList.Add(item.expect); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } catch (Exception ex) { log.Error(typeof(BJKL8Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } public class bjkl8Model { public string expect { get; set; } public string opencode { get; set; } public string opentime { get; set; } } /// /// 通过备用站点抓取开奖数据 /// (北京福彩官网) /// private void DoTodayJobByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl)) { var OpenList = GetOpenListFromBackUrl(Config.BackUrl); if (OpenList.Count == 0) return;//无抓取数据 var startQiNum = LatestItem.Term + 1; var newestQiNum = OpenList.First().Term; if (startQiNum > newestQiNum) return;//无最新数据 //处理最新开奖数据 for (var i = startQiNum; i <= newestQiNum; i++) { var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault(); if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(BJKL8Job), CommonHelper.GetJobBackLogInfo(Config, i.ToString())); LatestItem = matchItem; } } } } /// /// 通过备用地址抓取错误期号列表中每一个期号 /// (北京福彩官网) /// private void DoYesterdayFailedListByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetOpenListFromBackUrl(Config.BackUrl); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (var failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault(); if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(BJKL8Job), CommonHelper.GetJobBackLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } //foreach (OpenCode21Model model in OpenList) //{ // if (services.AddOpen21Code(currentLottery, model)) // { // //处理成功写入日志 // log.Info(typeof(BJKL8Job), CommonHelper.GetJobBackLogInfo(Config, model.Term.ToString())); // } //} } } /// /// 抓取备用站点开奖数据 /// 由于北京福彩官网分页30条数据一页,而此彩种每天开179期,如果去跑6页,负荷太大,只有通过主站点来保证数据完整性 /// /// 备用站点 /// private List GetOpenListFromBackUrl(string url) { List result = new List(); try { string HtmlResource = string.Empty; HtmlResource = NetHelper.GetUrlResponse(url); if (!string.IsNullOrWhiteSpace(HtmlResource)) { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(HtmlResource); var table = doc.DocumentNode.SelectNodes("//table"); if (table == null || table.Count < 2) return result; var trs = table[1].ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList(); if (trs.Count < 2) return result; List tds = null; OpenCode21Model model = null; string[] openCodeList = null; string openTime = string.Empty; string todayDateString = CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd"); for (var i = 1; i < trs.Count; i++)//第一行是表头 { tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 4) continue; openTime = tds[3].InnerText.Trim(); if (!openTime.StartsWith(todayDateString)) continue; model = new OpenCode21Model(); model.Term = Convert.ToInt64(tds[0].InnerText.Trim()); model.OpenCode21 = Convert.ToInt32(tds[2].InnerText.Trim());//飞盘 openCodeList = tds[1].InnerText.Trim().Split(new char[] { ',' }); if (openCodeList.Length < 20) continue; model.OpenCode1 = Convert.ToInt32(openCodeList[0]); model.OpenCode2 = Convert.ToInt32(openCodeList[1]); model.OpenCode3 = Convert.ToInt32(openCodeList[2]); model.OpenCode4 = Convert.ToInt32(openCodeList[3]); model.OpenCode5 = Convert.ToInt32(openCodeList[4]); model.OpenCode6 = Convert.ToInt32(openCodeList[5]); model.OpenCode7 = Convert.ToInt32(openCodeList[6]); model.OpenCode8 = Convert.ToInt32(openCodeList[7]); model.OpenCode9 = Convert.ToInt32(openCodeList[8]); model.OpenCode10 = Convert.ToInt32(openCodeList[9]); model.OpenCode11 = Convert.ToInt32(openCodeList[10]); model.OpenCode12 = Convert.ToInt32(openCodeList[11]); model.OpenCode13 = Convert.ToInt32(openCodeList[12]); model.OpenCode14 = Convert.ToInt32(openCodeList[13]); model.OpenCode15 = Convert.ToInt32(openCodeList[14]); model.OpenCode16 = Convert.ToInt32(openCodeList[15]); model.OpenCode17 = Convert.ToInt32(openCodeList[16]); model.OpenCode18 = Convert.ToInt32(openCodeList[17]); model.OpenCode19 = Convert.ToInt32(openCodeList[18]); model.OpenCode20 = Convert.ToInt32(openCodeList[19]); model.OpenTime = Convert.ToDateTime(openTime); if (!result.Contains(model)) result.Add(model); } } } catch (Exception ex) { log.Error(typeof(BJKL8Job), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 生成今日第一期记录 /// /// private OpenCode21Model GenerateTodayFirstItem() { OpenCode21Model item = new OpenCode21Model(); var currentDateTime = CommonHelper.SCCSysDateTime; TimeSpan datepart = currentDateTime - new DateTime(2017, 1, 1); var t = Config.SkipDate.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); item.Term = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear-6;//期号使用昨日最后一期期号,便于计算使用 item.OpenTime = new DateTime(currentDateTime.Year, currentDateTime.Month, currentDateTime.Day, Config.StartHour, Config.StartMinute, 0); return item; } /// /// 核实最新一期开奖记录是否为今日最后一期 /// /// 期号 /// private bool CheckLatestItemIsTodayLastItem() { if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd")) return true; var firstItem = GenerateTodayFirstItem(); if (LatestItem.Term == (firstItem.Term + Config.TimesPerDay)) return true; return false; } /// /// 获取今天第一期期号 /// /// private string GenerateTodayFirstQiHao() { TimeSpan datepart = CommonHelper.SCCSysDateTime - new DateTime(2017, 1, 1); var t = Config.SkipDate.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries); var terms = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear + 1; return terms.ToString(); } /// /// 生成昨天的期号列表 /// /// private List GenerateYesterdayQiHaoList() { List result = new List(); var todayFirstQiHao = Convert.ToInt64(GenerateTodayFirstQiHao()); for (var i = Config.TimesPerDay; i > 0; i--) { result.Add((todayFirstQiHao - i).ToString()); } return result; } /// /// 生成快乐8的开奖时间 /// /// /// private DateTime GenerateYesterdayKL8OpenTime(string QiHao) { var openDay = CommonHelper.SCCSysDateTime.AddDays(-1); var StartTime = new DateTime(openDay.Year, openDay.Month, openDay.Day, Config.StartHour, Config.StartMinute, 0); var t = (Convert.ToInt64(QiHao) - LastTermLastYear) % Config.TimesPerDay; if (t == 0) return StartTime.AddMinutes((Config.TimesPerDay - 1) * Config.Interval); else return StartTime.AddMinutes((t - 1) * Config.Interval); } #region Attribute /// /// 配置信息 /// private SCCConfig Config = null; /// /// 当天抓取的最新一期开奖记录 /// private OpenCode21Model LatestItem = null; /// /// 当天抓取失败列表 /// private List FailedQiHaoList = null; /// /// 日志对象 /// private LogHelper log = null; /// /// 数据服务 /// private IOpen21Code services = null; /// /// 当前彩种 /// private SCCLottery currentLottery { get { return SCCLottery.BeiJingKL8; } } /// /// 邮件接口 /// private IEmail email = null; /// /// 2016年最后一期期号 /// private long LastTermLastYear = 800589; #endregion } }