using System; using System.Collections.Generic; using System.Linq; using System.Text; using Quartz; using SCC.Common; using SCC.Models; using SCC.Interface; using HtmlAgilityPack; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace SCC.Crawler.GP { /// /// 数据爬取类 /// 吉林快3 /// [DisallowConcurrentExecution] [PersistJobDataAfterExecution] public class JLK3Job : IJob { /// /// 构造函数 /// public JLK3Job() { log = new LogHelper(); services = IOC.Resolve(); email = IOC.Resolve(); } /// /// 作业执行入口 /// /// 作业执行上下文 public void Execute(IJobExecutionContext context) { Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap); //预设节假日不开奖 if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return; LatestQiHao = context.JobDetail.JobDataMap.GetString("LatestQiHao"); try { //服务启动时配置初始数据 if (string.IsNullOrEmpty(LatestQiHao)) { var lastItem = services.GetLastItem(currentLottery); if (lastItem != null) { LatestQiHao = lastItem.Term.ToString(); } } //第一次启动服务或最新期号为昨天的开奖期号,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表 if (string.IsNullOrEmpty(LatestQiHao) || !LatestQiHao.StartsWith(CommonHelper.SCCSysDateTime.ToString("yyMMdd"))) { CheckingYesterdayTheLotteryData(); LatestQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(0); } //当最新期号不符合当天总期数,执行当天作业 if (Convert.ToInt32(LatestQiHao.Substring(6)) != Config.TimesPerDay) { DoTodayJobByMainUrl(); DoTodayJobByBackUrl(); } } catch (Exception ex) { log.Error(typeof(JLK3Job), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } //保存最新期号和失败期号列表 context.JobDetail.JobDataMap["LatestQiHao"] = LatestQiHao; } /// /// 自检昨天开奖数据 /// private void CheckingYesterdayTheLotteryData() { if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据 //从数据库中获取昨天数据抓取失败列表 FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, Config.TimesPerDay); if (FailedQiHaoList.Count > 0) { DoYesterdayFailedListByMainUrl(); DoYesterdayFailedListByBackUrl(); foreach (var fQiHao in FailedQiHaoList) { //将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中 if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, CommonHelper.GenerateYesterdayOpenTime(Config, fQiHao))) log.Error(typeof(JLK3Job), CommonHelper.GetJobLogError(Config, fQiHao)); } } } /// /// 通过主站点抓取开奖数据 /// (爱彩乐) /// private void DoTodayJobByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl)) { var OpenList = GetTodayOpenListFromMainUrl(); if (OpenList.Count == 0) return;//无抓取数据 var newestQiHao = OpenList.First().Key; var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1; var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6)); if (startQiNum > newestQiNum) return;//无最新数据 //处理最新开奖数据 string getQiHao = string.Empty; for (var i = startQiNum; i <= newestQiNum; i++) { getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i); var matchItem = OpenList.Where(R => R.Key == getQiHao).FirstOrDefault(); if (matchItem.Key != null && SaveRecord(getQiHao, matchItem.Value, false)) { //处理成功写入日志 log.Info(typeof(JLK3Job), CommonHelper.GetJobMainLogInfo(Config, getQiHao)); LatestQiHao = getQiHao; } } } } /// /// 通过主站抓取错误期号列表中每一个期号 /// (爱彩乐) /// private void DoYesterdayFailedListByMainUrl() { if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetYesterdayOpenListFromMainUrl(); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (string failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault(); if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true)) { //处理成功写入日志 log.Info(typeof(JLK3Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } /// /// 通过主站点抓取今日最新开奖列表 /// /// private Dictionary GetTodayOpenListFromMainUrl() { Dictionary result = new Dictionary(); try { var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(HtmlResource); var table = doc.DocumentNode.SelectSingleNode("//table"); if (table == null) return result; var trs = table.ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList(); List tds = null, ems = null; var matchQiHao = string.Empty; var matchKJHaoMa = string.Empty; for (var i = 1; i < trs.Count; i++)//第一行为表头 { tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList(); if (tds.Count < 3) continue; matchQiHao = tds[0].InnerText.Trim(); ems = tds[2].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList(); if (ems.Count < 3) continue; matchKJHaoMa = string.Format("{0},{1},{2}", ems[0].InnerText.Trim(), ems[1].InnerText.Trim(), ems[2].InnerText.Trim()); if (!result.ContainsKey(matchQiHao)) result.Add(matchQiHao, matchKJHaoMa); } } } catch (Exception ex) { log.Error(typeof(JLK3Job), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过主站点抓取昨日开奖列表 /// /// private Dictionary GetYesterdayOpenListFromMainUrl() { Dictionary result = new Dictionary(); try { var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl + "?action=chart&date=yesterday&id=806&async=true"); if (!string.IsNullOrWhiteSpace(HtmlResource)) { var obj = JsonConvert.DeserializeObject(HtmlResource); if (obj != null && obj["data"] != null) { var matchQiHao = string.Empty; var matchKJHaoMa = string.Empty; JArray openCodeList = null; foreach (var item in obj["data"]) { matchQiHao = item["dateNumber"].ToString(); openCodeList = (JArray)item["list"]; matchKJHaoMa = string.Format("{0},{1},{2}", openCodeList[0].ToString(), openCodeList[1].ToString(), openCodeList[2].ToString()); if (!result.ContainsKey(matchQiHao)) result.Add(matchQiHao, matchKJHaoMa); } } } } catch (Exception ex) { log.Error(typeof(JLK3Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 通过备用站点抓取开奖数据 /// (彩乐乐) /// private void DoTodayJobByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl)) { var OpenList = GetTodayOpenListFromBackUrl(); if (OpenList.Count == 0) return;//无抓取数据 var newestQiHao = OpenList.Last().Key; var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1; var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6)); if (startQiNum > newestQiNum) return;//无最新数据 //处理最新开奖数据 var getQiHao = string.Empty; for (var i = startQiNum; i <= newestQiNum; i++) { getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i); var matchItem = OpenList.Where(R => R.Key == getQiHao).FirstOrDefault(); if (matchItem.Key != null && SaveRecord(getQiHao, matchItem.Value, false)) { //处理成功写入日志 log.Info(typeof(JLK3Job), CommonHelper.GetJobBackLogInfo(Config, getQiHao)); LatestQiHao = getQiHao; } } } } /// /// 通过备用地址抓取错误期号列表中每一个期号 /// (彩乐乐) /// private void DoYesterdayFailedListByBackUrl() { if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0) { var OpenList = GetYesterdayOpenListFromBackUrl(); if (OpenList.Count == 0) return;//无抓取数据 var SuccessList = new List(); foreach (var failedQiHao in FailedQiHaoList) { var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault(); if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true)) { //处理成功写入日志 log.Info(typeof(JLK3Job), CommonHelper.GetJobBackLogInfo(Config, failedQiHao)); SuccessList.Add(failedQiHao); continue; } } foreach (var successQiHao in SuccessList) { FailedQiHaoList.Remove(successQiHao); } } } /// /// 通过备用站点抓取今日最新开奖数据 /// /// private Dictionary GetTodayOpenListFromBackUrl() { var requestUrl = string.Format(Config.BackUrl, 0); return GetOpenListFromBackUrl(requestUrl); } /// /// 通过备用站点抓取昨日开奖数据 /// /// private Dictionary GetYesterdayOpenListFromBackUrl() { var requestUrl = string.Format(Config.BackUrl, 1); return GetOpenListFromBackUrl(requestUrl); } /// /// 通过备用站点抓取开奖数据 /// /// 备用站点 /// private Dictionary GetOpenListFromBackUrl(string requestUrl) { Dictionary result = new Dictionary(); try { var HtmlResource = NetHelper.GetUrlResponse(requestUrl); if (!string.IsNullOrWhiteSpace(HtmlResource)) { HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(HtmlResource); var tbody = doc.GetElementbyId("cpdata_new"); if (tbody == null) return result; var trs = tbody.ChildNodes.Where(node => node.Name == "tr").ToList(); string matchQiHao = string.Empty; string matchKJHaoMa = string.Empty; for (var i = 0; i < trs.Count; i++)//第一行是表头 { var trstyle = trs[i].Attributes["style"]; if (trstyle != null && trstyle.Value == "display:none") { continue; } var tds = trs[i].ChildNodes.Where(node => node.Name == "td").ToList(); matchQiHao = tds[0].InnerText.Trim(); matchKJHaoMa = string.Format("{0},{1},{2}", tds[1].InnerText.Trim(), tds[2].InnerText.Trim(), tds[3].InnerText.Trim()); result.Add(matchQiHao, matchKJHaoMa); } } } catch (Exception ex) { log.Error(typeof(JLK3Job), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message)); } return result; } /// /// 将此彩种指定期号和开奖号码保存至数据库 /// /// 期号 /// 开奖号码(形如01,02,03) /// 是否是保存昨天的记录 /// private bool SaveRecord(string QiHao, string OpenCode, bool IsYesterdayRecord) { if (!string.IsNullOrWhiteSpace(QiHao) && !string.IsNullOrWhiteSpace(OpenCode)) { OpenCode3Model model = new OpenCode3Model(); model.Term = Convert.ToInt64(QiHao); var haoMaArray = OpenCode.Split(new char[] { ',' }); model.OpenCode1 = Convert.ToInt32(haoMaArray[0]); model.OpenCode2 = Convert.ToInt32(haoMaArray[1]); model.OpenCode3 = Convert.ToInt32(haoMaArray[2]); if (IsYesterdayRecord) model.OpenTime = CommonHelper.GenerateYesterdayOpenTime(Config, QiHao); else model.OpenTime = CommonHelper.GenerateTodayOpenTime(Config, QiHao); return services.AddOpen3Code(currentLottery, model); } return false; } #region Attribute /// /// 配置信息 /// private SCCConfig Config = null; /// /// 当天抓取的最新一期期号 /// private string LatestQiHao = null; /// /// 当天抓取失败列表 /// private List FailedQiHaoList = null; /// /// 日志对象 /// private LogHelper log = null; /// /// 数据服务 /// private IOpen3Code services = null; /// /// 当前彩种 /// private SCCLottery currentLottery { get { return SCCLottery.JiLinK3; } } /// /// 邮件接口 /// private IEmail email = null; #endregion } }