using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Xml; using Quartz; using SCC.Common; using SCC.Models; using SCC.Interface; using HtmlAgilityPack; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace SCC.Crawler.GP { /// /// 数据爬取类 /// 重庆快乐十分 /// [DisallowConcurrentExecution] [PersistJobDataAfterExecution] public class CQKL10FJob : IJob { /// /// 构造函数 /// public CQKL10FJob() { log = new LogHelper(); services = IOC.Resolve(); email = IOC.Resolve(); } /// /// 作业执行入口 /// /// 作业执行上下文 public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); //预设节假日不开奖 if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return; LatestQiHao = context.JobDetail.JobDataMap.GetString("LatestQiHao"); try { //服务启动时配置初始数据 if (string.IsNullOrEmpty(LatestQiHao)) { var lastItem = services.GetLastItem(currentLottery); if (lastItem != null) { LatestQiHao = lastItem.Term.ToString(); } } //第一次启动服务或最新期号为昨天的开奖期号,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表 if (string.IsNullOrEmpty(LatestQiHao) || !LatestQiHao.StartsWith(CommonHelper.SCCSysDateTime.ToString("yyMMdd"))) { CheckingYesterdayTheLotteryData(); LatestQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(0); } //当最新期号不符合当天总期数,执行当天作业 if (Convert.ToInt32(LatestQiHao.Substring(6)) != Config.TimesPerDay) { DoTodayJobByMainUrl(); //DoTodayJobByBackUrl(); } } catch (Exception ex) { log.Error(typeof(CQKL10FJob), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } //保存最新期号和失败期号列表 context.JobDetail.JobDataMap["LatestQiHao"] = LatestQiHao; } /// /// 自检昨天开奖数据 /// private void CheckingYesterdayTheLotteryData() { if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据 //从数据库中获取昨天数据抓取失败列表 FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, Config.TimesPerDay); if (FailedQiHaoList.Count > 0) { DoYesterdayFailedListByMainUrl_New(); //DoYesterdayFailedListByMainUrl(); //DoYesterdayFailedListByBackUrl(); foreach (var fQiHao in FailedQiHaoList) { //将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中 if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, CommonHelper.GenerateYesterdayOpenTime(Config, fQiHao))) log.Error(typeof(CQKL10FJob), CommonHelper.GetJobLogError(Config, fQiHao)); } } } /// /// 通过接口抓取错误期号列表中每一个期号 /// (爱彩乐) /// private void DoYesterdayFailedListByMainUrl_New() { if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0) { var html = new HttpHelper().PostJson(new Dictionary() { { "SiteId" , Config.SiteId }, { "Date",DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd 00:00:00")}, }, Config.BackUrl); string HtmlResource = html; JObject obj = JsonConvert.DeserializeObject(html); var s = obj["Data"]; JObject obj1 = JsonConvert.DeserializeObject(s.ToString()); var s1 = obj1["LotteryResults"]; var OpenList = JsonConvert.DeserializeObject>(s1.ToString()); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (string failedQiHao in FailedQiHaoList) { OpenCode8Model model = new OpenCode8Model(); var matchItem = OpenList.Where(R => R.PeriodNo.Replace("-", "0") == failedQiHao).FirstOrDefault(); string value = ""; foreach (var item in matchItem.LotteryNumbers) { value += item.Number + ","; } model.Term = Convert.ToInt64(matchItem.PeriodNo.Replace("-", "0")); model.OpenCode1 = Convert.ToInt32(matchItem.LotteryNumbers[0].Number); model.OpenCode2 = Convert.ToInt32(matchItem.LotteryNumbers[1].Number); model.OpenCode3 = Convert.ToInt32(matchItem.LotteryNumbers[2].Number); model.OpenCode4 = Convert.ToInt32(matchItem.LotteryNumbers[3].Number); model.OpenCode5 = Convert.ToInt32(matchItem.LotteryNumbers[4].Number); model.OpenCode6 = Convert.ToInt32(matchItem.LotteryNumbers[5].Number); model.OpenCode7 = Convert.ToInt32(matchItem.LotteryNumbers[6].Number); model.OpenCode8 = Convert.ToInt32(matchItem.LotteryNumbers[7].Number); string d = "20" + matchItem.PeriodNo.Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8,2) + " " + matchItem.Thedatetime; model.OpenTime = Convert.ToDateTime("20" + matchItem.PeriodNo.Replace("-", "").Insert(2,"-").Insert(5,"-").Remove(8, 2) + " " + matchItem.Thedatetime); if (matchItem.PeriodNo != null && services.AddOpen8Code(currentLottery, model)) { //处理成功写入日志 log.Info(typeof(AH11X5Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } /// /// 通过主站点抓取开奖数据 /// (重庆福彩官网) /// private void DoTodayJobByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl)) { var OpenList = GetTodayOpenListFromMainUrl(); if (OpenList.Count == 0) return;//无抓取数据 var newestQiHao = OpenList.Last().Term.ToString(); var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1; var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6)); if (startQiNum > newestQiNum) return;//无最新数据 //处理最新开奖数据 string getQiHao = string.Empty; for (var i = startQiNum; i <= newestQiNum; i++) { getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i); var matchItem = OpenList.Where(R => R.Term.ToString() == getQiHao).FirstOrDefault(); if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(CQKL10FJob), CommonHelper.GetJobMainLogInfo(Config, getQiHao)); LatestQiHao = getQiHao; } } } } /// /// 通过主站抓取错误期号列表中每一个期号 /// (重庆福彩官网) /// private void DoYesterdayFailedListByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetYesterdayOpenListFromMainUrl(); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (string failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault(); if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(CQKL10FJob), CommonHelper.GetJobMainLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } /// /// 通过主站点抓取今日最新开奖数据 /// /// private List GetTodayOpenListFromMainUrl() { List result = new List(); try { var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { HtmlDocument doc = new HtmlDocument(); OpenCode8Model model = null; doc.LoadHtml(HtmlResource); var table = doc.DocumentNode.SelectNodes("//*[@id='firstTbody']/tr"); if (table == null) return result; var s = table.ToList(); var trs = table.ToList(); List tds = null, fonts = null; var matchQiHao = string.Empty; var matchKJHaoMa = string.Empty; for (var i = 0; i < trs.Count; i++)//第一行为表头 { model = new OpenCode8Model(); tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 3) continue; matchQiHao = tds[0].InnerText.Trim().Replace("-", "0"); model.Term = Convert.ToInt64(matchQiHao); fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList(); if (fonts.Count < 8) continue; model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Trim()); model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Trim()); model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Trim()); model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Trim()); model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Trim()); model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Trim()); model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Trim()); model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Trim()); model.OpenTime = Convert.ToDateTime("20" + tds[0].InnerText.Trim().Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8, 2) + " " + tds[2].InnerText.Trim()); if (!result.Contains(model)) result.Add(model); } table = doc.DocumentNode.SelectNodes("//*[@id='secondTbody']/tr"); if (table == null) return result; trs = table.ToList(); for (var i = 0; i < trs.Count; i++)//第一行为表头 { model = new OpenCode8Model(); tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 3) continue; matchQiHao = tds[0].InnerText.Trim().Replace("-", ""); model.Term = Convert.ToInt64(matchQiHao); fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList(); if (fonts.Count < 8) continue; model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Trim()); model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Trim()); model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Trim()); model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Trim()); model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Trim()); model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Trim()); model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Trim()); model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Trim()); model.OpenTime = Convert.ToDateTime("20" + tds[0].InnerText.Trim().Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8, 2) + " " + tds[2].InnerText.Trim()); if (!result.Contains(model)) result.Add(model); } table = doc.DocumentNode.SelectNodes("//*[@id='thirdTbody']/tr"); if (table == null) return result; trs = table.ToList(); for (var i = 0; i < trs.Count; i++)//第一行为表头 { model = new OpenCode8Model(); tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 3) continue; matchQiHao = tds[0].InnerText.Trim().Replace("-", ""); model.Term = Convert.ToInt64(matchQiHao); fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList(); if (fonts.Count < 8) continue; model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Trim()); model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Trim()); model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Trim()); model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Trim()); model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Trim()); model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Trim()); model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Trim()); model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Trim()); model.OpenTime = Convert.ToDateTime("20" + tds[0].InnerText.Trim().Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8, 2) + " " + tds[2].InnerText.Trim()); if (!result.Contains(model)) result.Add(model); } } } catch (Exception ex) { log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过主站点抓取昨日开奖数据 /// /// private List GetYesterdayOpenListFromMainUrl() { List result = new List(); try { for (var k = 1; k <= 2; k++) { var HtmlResource = NetHelper.GetCQKL10FUrlResponse(Config.MainUrl, k); if (!string.IsNullOrWhiteSpace(HtmlResource)) { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(HtmlResource); var table = doc.DocumentNode.SelectSingleNode("//table"); if (table == null) return result; var trs = table.ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList(); if (trs.Count < 3) return result; List tds = null, fonts = null; OpenCode8Model model = null; for (var i = 1; i < trs.Count - 1; i++)//第一行是表头,最后一行是页码 { tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 8) continue; model = new OpenCode8Model(); model.Term = Convert.ToInt64(tds[0].InnerText.Trim().Substring(2)); fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "font").ToList(); if (fonts.Count < 8) continue; model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Replace(" ", string.Empty).Trim()); model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Replace(" ", string.Empty).Trim()); model.OpenTime = Convert.ToDateTime(tds[7].InnerText.Replace(" ", " ").Trim()); if (!result.Contains(model)) result.Add(model); } } } } catch (Exception ex) { log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过备用站点抓取开奖数据 /// (爱彩乐) /// private void DoTodayJobByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl)) { var OpenList = GetTodayOpenListFromBackUrl(); if (OpenList.Count == 0) return;//无抓取数据 var newestQiHao = OpenList.First().Term.ToString(); var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1; var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6)); if (startQiNum > newestQiNum) return;//无最新数据 //处理最新开奖数据 var getQiHao = string.Empty; for (var i = startQiNum; i <= newestQiNum; i++) { getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i); var matchItem = OpenList.Where(R => R.Term.ToString() == getQiHao).FirstOrDefault(); if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(CQKL10FJob), CommonHelper.GetJobBackLogInfo(Config, getQiHao)); LatestQiHao = getQiHao; } } } } /// /// 通过备用地址抓取错误期号列表中每一个期号 /// (爱彩乐) /// private void DoYesterdayFailedListByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetYesterdayOpenListFromBackUrl(); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (var failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault(); if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem)) { //处理成功写入日志 log.Info(typeof(CQKL10FJob), CommonHelper.GetJobBackLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } /// /// 通过备用站点抓取今日最新开奖数据 /// /// private List GetTodayOpenListFromBackUrl() { List result = new List(); try { var HtmlResource = NetHelper.GetUrlResponse(Config.BackUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(HtmlResource); var table = doc.DocumentNode.SelectSingleNode("//table"); if (table == null) return result; var trs = table.ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList(); List tds = null, ems = null; var matchQiHao = string.Empty; var matchKJHaoMa = string.Empty; OpenCode8Model model = null; for (var i = 1; i < trs.Count; i++)//第一行为表头 { tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 3) continue; model = new OpenCode8Model(); model.Term = Convert.ToInt64(tds[0].InnerText.Trim().Insert(6, "0")); ems = tds[2].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList(); if (ems.Count < 8) continue; model.OpenCode1 = Convert.ToInt32(ems[0].InnerText.Trim()); model.OpenCode2 = Convert.ToInt32(ems[1].InnerText.Trim()); model.OpenCode3 = Convert.ToInt32(ems[2].InnerText.Trim()); model.OpenCode4 = Convert.ToInt32(ems[3].InnerText.Trim()); model.OpenCode5 = Convert.ToInt32(ems[4].InnerText.Trim()); model.OpenCode6 = Convert.ToInt32(ems[5].InnerText.Trim()); model.OpenCode7 = Convert.ToInt32(ems[6].InnerText.Trim()); model.OpenCode8 = Convert.ToInt32(ems[7].InnerText.Trim()); model.OpenTime = Convert.ToDateTime(tds[1].InnerText.Trim()); if (!result.Contains(model)) result.Add(model); } } } catch (Exception ex) { log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过备用站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过备用站点抓取昨日开奖数据 /// /// private List GetYesterdayOpenListFromBackUrl() { List result = new List(); try { var HtmlResource = NetHelper.GetUrlResponse(Config.BackUrl + "?action=chart&date=yesterday&id=1006&async=true"); if (!string.IsNullOrWhiteSpace(HtmlResource)) { var obj = JsonConvert.DeserializeObject(HtmlResource); if (obj != null && obj["data"] != null) { JArray openCodeList = null; OpenCode8Model model = null; foreach (var item in obj["data"]) { model = new OpenCode8Model(); model.Term = Convert.ToInt64(item["dateNumber"].ToString()); model.OpenTime = Convert.ToDateTime(item["dateTime"].ToString()); openCodeList = (JArray)item["list"]; model.OpenCode1 = Convert.ToInt32(openCodeList[0].ToString()); model.OpenCode2 = Convert.ToInt32(openCodeList[1].ToString()); model.OpenCode3 = Convert.ToInt32(openCodeList[2].ToString()); model.OpenCode4 = Convert.ToInt32(openCodeList[3].ToString()); model.OpenCode5 = Convert.ToInt32(openCodeList[4].ToString()); model.OpenCode6 = Convert.ToInt32(openCodeList[5].ToString()); model.OpenCode7 = Convert.ToInt32(openCodeList[6].ToString()); model.OpenCode8 = Convert.ToInt32(openCodeList[7].ToString()); if (!result.Contains(model)) result.Add(model); } } } } catch (Exception ex) { log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过备用站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } #region Attribute /// /// 配置信息 /// private SCCConfig Config = null; /// /// 当天抓取的最新一期期号 /// private string LatestQiHao = null; /// /// 当天抓取失败列表 /// private List FailedQiHaoList = null; /// /// 日志对象 /// private LogHelper log = null; /// /// 数据服务 /// private IOpen8Code services = null; /// /// 当前彩种 /// private SCCLottery currentLottery { get { return SCCLottery.ChongQingKL10F; } } /// /// 邮件接口 /// private IEmail email = null; #endregion } }