using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Quartz;
using SCC.Common;
using SCC.Models;
using SCC.Interface;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace SCC.Crawler.GP
{
///
/// 数据爬取类
/// 重庆快乐十分
///
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class CQKL10FJob : IJob
{
///
/// 构造函数
///
public CQKL10FJob()
{
log = new LogHelper();
services = IOC.Resolve();
email = IOC.Resolve();
}
///
/// 作业执行入口
///
/// 作业执行上下文
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
//预设节假日不开奖
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
LatestQiHao = context.JobDetail.JobDataMap.GetString("LatestQiHao");
try
{
//服务启动时配置初始数据
if (string.IsNullOrEmpty(LatestQiHao))
{
var lastItem = services.GetLastItem(currentLottery);
if (lastItem != null)
{
LatestQiHao = lastItem.Term.ToString();
}
}
//第一次启动服务或最新期号为昨天的开奖期号,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表
if (string.IsNullOrEmpty(LatestQiHao) || !LatestQiHao.StartsWith(CommonHelper.SCCSysDateTime.ToString("yyMMdd")))
{
CheckingYesterdayTheLotteryData();
LatestQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(0);
}
//当最新期号不符合当天总期数,执行当天作业
if (Convert.ToInt32(LatestQiHao.Substring(6)) != Config.TimesPerDay)
{
DoTodayJobByMainUrl();
//DoTodayJobByBackUrl();
}
}
catch (Exception ex)
{
log.Error(typeof(CQKL10FJob), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
//保存最新期号和失败期号列表
context.JobDetail.JobDataMap["LatestQiHao"] = LatestQiHao;
}
///
/// 自检昨天开奖数据
///
private void CheckingYesterdayTheLotteryData()
{
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据
//从数据库中获取昨天数据抓取失败列表
FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, Config.TimesPerDay);
if (FailedQiHaoList.Count > 0)
{
DoYesterdayFailedListByMainUrl_New();
//DoYesterdayFailedListByMainUrl();
//DoYesterdayFailedListByBackUrl();
foreach (var fQiHao in FailedQiHaoList)
{
//将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中
if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, CommonHelper.GenerateYesterdayOpenTime(Config, fQiHao)))
log.Error(typeof(CQKL10FJob), CommonHelper.GetJobLogError(Config, fQiHao));
}
}
}
///
/// 通过接口抓取错误期号列表中每一个期号
/// (爱彩乐)
///
private void DoYesterdayFailedListByMainUrl_New()
{
if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
{
var html = new HttpHelper().PostJson(new Dictionary()
{
{ "SiteId" , Config.SiteId },
{ "Date",DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd 00:00:00")},
}, Config.BackUrl);
string HtmlResource = html;
JObject obj = JsonConvert.DeserializeObject(html);
var s = obj["Data"];
JObject obj1 = JsonConvert.DeserializeObject(s.ToString());
var s1 = obj1["LotteryResults"];
var OpenList = JsonConvert.DeserializeObject>(s1.ToString());
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
OpenCode8Model model = new OpenCode8Model();
var matchItem = OpenList.Where(R => R.PeriodNo.Replace("-", "0") == failedQiHao).FirstOrDefault();
string value = "";
foreach (var item in matchItem.LotteryNumbers)
{
value += item.Number + ",";
}
model.Term = Convert.ToInt64(matchItem.PeriodNo.Replace("-", "0"));
model.OpenCode1 = Convert.ToInt32(matchItem.LotteryNumbers[0].Number);
model.OpenCode2 = Convert.ToInt32(matchItem.LotteryNumbers[1].Number);
model.OpenCode3 = Convert.ToInt32(matchItem.LotteryNumbers[2].Number);
model.OpenCode4 = Convert.ToInt32(matchItem.LotteryNumbers[3].Number);
model.OpenCode5 = Convert.ToInt32(matchItem.LotteryNumbers[4].Number);
model.OpenCode6 = Convert.ToInt32(matchItem.LotteryNumbers[5].Number);
model.OpenCode7 = Convert.ToInt32(matchItem.LotteryNumbers[6].Number);
model.OpenCode8 = Convert.ToInt32(matchItem.LotteryNumbers[7].Number);
string d = "20" + matchItem.PeriodNo.Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8,2) + " " + matchItem.Thedatetime;
model.OpenTime = Convert.ToDateTime("20" + matchItem.PeriodNo.Replace("-", "").Insert(2,"-").Insert(5,"-").Remove(8, 2) + " " + matchItem.Thedatetime);
if (matchItem.PeriodNo != null && services.AddOpen8Code(currentLottery, model))
{
//处理成功写入日志
log.Info(typeof(AH11X5Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过主站点抓取开奖数据
/// (重庆福彩官网)
///
private void DoTodayJobByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl))
{
var OpenList = GetTodayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.Last().Term.ToString();
var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1;
var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6));
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
string getQiHao = string.Empty;
for (var i = startQiNum; i <= newestQiNum; i++)
{
getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i);
var matchItem = OpenList.Where(R => R.Term.ToString() == getQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(CQKL10FJob), CommonHelper.GetJobMainLogInfo(Config, getQiHao));
LatestQiHao = getQiHao;
}
}
}
}
///
/// 通过主站抓取错误期号列表中每一个期号
/// (重庆福彩官网)
///
private void DoYesterdayFailedListByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(CQKL10FJob), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过主站点抓取今日最新开奖数据
///
///
private List GetTodayOpenListFromMainUrl()
{
List result = new List();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
OpenCode8Model model = null;
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectNodes("//*[@id='firstTbody']/tr");
if (table == null) return result;
var s = table.ToList();
var trs = table.ToList();
List tds = null, fonts = null;
var matchQiHao = string.Empty;
var matchKJHaoMa = string.Empty;
for (var i = 0; i < trs.Count; i++)//第一行为表头
{
model = new OpenCode8Model();
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 3) continue;
matchQiHao = tds[0].InnerText.Trim().Replace("-", "0");
model.Term = Convert.ToInt64(matchQiHao);
fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList();
if (fonts.Count < 8) continue;
model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Trim());
model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Trim());
model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Trim());
model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Trim());
model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Trim());
model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Trim());
model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Trim());
model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Trim());
model.OpenTime = Convert.ToDateTime("20" + tds[0].InnerText.Trim().Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8, 2) + " " + tds[2].InnerText.Trim());
if (!result.Contains(model))
result.Add(model);
}
table = doc.DocumentNode.SelectNodes("//*[@id='secondTbody']/tr");
if (table == null) return result;
trs = table.ToList();
for (var i = 0; i < trs.Count; i++)//第一行为表头
{
model = new OpenCode8Model();
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 3) continue;
matchQiHao = tds[0].InnerText.Trim().Replace("-", "");
model.Term = Convert.ToInt64(matchQiHao);
fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList();
if (fonts.Count < 8) continue;
model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Trim());
model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Trim());
model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Trim());
model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Trim());
model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Trim());
model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Trim());
model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Trim());
model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Trim());
model.OpenTime = Convert.ToDateTime("20" + tds[0].InnerText.Trim().Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8, 2) + " " + tds[2].InnerText.Trim());
if (!result.Contains(model))
result.Add(model);
}
table = doc.DocumentNode.SelectNodes("//*[@id='thirdTbody']/tr");
if (table == null) return result;
trs = table.ToList();
for (var i = 0; i < trs.Count; i++)//第一行为表头
{
model = new OpenCode8Model();
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 3) continue;
matchQiHao = tds[0].InnerText.Trim().Replace("-", "");
model.Term = Convert.ToInt64(matchQiHao);
fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList();
if (fonts.Count < 8) continue;
model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Trim());
model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Trim());
model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Trim());
model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Trim());
model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Trim());
model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Trim());
model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Trim());
model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Trim());
model.OpenTime = Convert.ToDateTime("20" + tds[0].InnerText.Trim().Replace("-", "").Insert(2, "-").Insert(5, "-").Remove(8, 2) + " " + tds[2].InnerText.Trim());
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过主站点抓取昨日开奖数据
///
///
private List GetYesterdayOpenListFromMainUrl()
{
List result = new List();
try
{
for (var k = 1; k <= 2; k++)
{
var HtmlResource = NetHelper.GetCQKL10FUrlResponse(Config.MainUrl, k);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectSingleNode("//table");
if (table == null) return result;
var trs = table.ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
if (trs.Count < 3) return result;
List tds = null, fonts = null;
OpenCode8Model model = null;
for (var i = 1; i < trs.Count - 1; i++)//第一行是表头,最后一行是页码
{
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 8) continue;
model = new OpenCode8Model();
model.Term = Convert.ToInt64(tds[0].InnerText.Trim().Substring(2));
fonts = tds[1].ChildNodes.Where(R => R.Name.ToLower() == "font").ToList();
if (fonts.Count < 8) continue;
model.OpenCode1 = Convert.ToInt32(fonts[0].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode2 = Convert.ToInt32(fonts[1].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode3 = Convert.ToInt32(fonts[2].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode4 = Convert.ToInt32(fonts[3].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode5 = Convert.ToInt32(fonts[4].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode6 = Convert.ToInt32(fonts[5].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode7 = Convert.ToInt32(fonts[6].InnerText.Replace(" ", string.Empty).Trim());
model.OpenCode8 = Convert.ToInt32(fonts[7].InnerText.Replace(" ", string.Empty).Trim());
model.OpenTime = Convert.ToDateTime(tds[7].InnerText.Replace(" ", " ").Trim());
if (!result.Contains(model))
result.Add(model);
}
}
}
}
catch (Exception ex)
{
log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过备用站点抓取开奖数据
/// (爱彩乐)
///
private void DoTodayJobByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl))
{
var OpenList = GetTodayOpenListFromBackUrl();
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.First().Term.ToString();
var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1;
var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6));
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
var getQiHao = string.Empty;
for (var i = startQiNum; i <= newestQiNum; i++)
{
getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i);
var matchItem = OpenList.Where(R => R.Term.ToString() == getQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(CQKL10FJob), CommonHelper.GetJobBackLogInfo(Config, getQiHao));
LatestQiHao = getQiHao;
}
}
}
}
///
/// 通过备用地址抓取错误期号列表中每一个期号
/// (爱彩乐)
///
private void DoYesterdayFailedListByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromBackUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (var failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen8Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(CQKL10FJob), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过备用站点抓取今日最新开奖数据
///
///
private List GetTodayOpenListFromBackUrl()
{
List result = new List();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.BackUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectSingleNode("//table");
if (table == null) return result;
var trs = table.ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
List tds = null, ems = null;
var matchQiHao = string.Empty;
var matchKJHaoMa = string.Empty;
OpenCode8Model model = null;
for (var i = 1; i < trs.Count; i++)//第一行为表头
{
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 3) continue;
model = new OpenCode8Model();
model.Term = Convert.ToInt64(tds[0].InnerText.Trim().Insert(6, "0"));
ems = tds[2].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList();
if (ems.Count < 8) continue;
model.OpenCode1 = Convert.ToInt32(ems[0].InnerText.Trim());
model.OpenCode2 = Convert.ToInt32(ems[1].InnerText.Trim());
model.OpenCode3 = Convert.ToInt32(ems[2].InnerText.Trim());
model.OpenCode4 = Convert.ToInt32(ems[3].InnerText.Trim());
model.OpenCode5 = Convert.ToInt32(ems[4].InnerText.Trim());
model.OpenCode6 = Convert.ToInt32(ems[5].InnerText.Trim());
model.OpenCode7 = Convert.ToInt32(ems[6].InnerText.Trim());
model.OpenCode8 = Convert.ToInt32(ems[7].InnerText.Trim());
model.OpenTime = Convert.ToDateTime(tds[1].InnerText.Trim());
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过备用站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过备用站点抓取昨日开奖数据
///
///
private List GetYesterdayOpenListFromBackUrl()
{
List result = new List();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.BackUrl + "?action=chart&date=yesterday&id=1006&async=true");
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
var obj = JsonConvert.DeserializeObject(HtmlResource);
if (obj != null && obj["data"] != null)
{
JArray openCodeList = null;
OpenCode8Model model = null;
foreach (var item in obj["data"])
{
model = new OpenCode8Model();
model.Term = Convert.ToInt64(item["dateNumber"].ToString());
model.OpenTime = Convert.ToDateTime(item["dateTime"].ToString());
openCodeList = (JArray)item["list"];
model.OpenCode1 = Convert.ToInt32(openCodeList[0].ToString());
model.OpenCode2 = Convert.ToInt32(openCodeList[1].ToString());
model.OpenCode3 = Convert.ToInt32(openCodeList[2].ToString());
model.OpenCode4 = Convert.ToInt32(openCodeList[3].ToString());
model.OpenCode5 = Convert.ToInt32(openCodeList[4].ToString());
model.OpenCode6 = Convert.ToInt32(openCodeList[5].ToString());
model.OpenCode7 = Convert.ToInt32(openCodeList[6].ToString());
model.OpenCode8 = Convert.ToInt32(openCodeList[7].ToString());
if (!result.Contains(model))
result.Add(model);
}
}
}
}
catch (Exception ex)
{
log.Error(typeof(CQKL10FJob), string.Format("【{0}】通过备用站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
#region Attribute
///
/// 配置信息
///
private SCCConfig Config = null;
///
/// 当天抓取的最新一期期号
///
private string LatestQiHao = null;
///
/// 当天抓取失败列表
///
private List FailedQiHaoList = null;
///
/// 日志对象
///
private LogHelper log = null;
///
/// 数据服务
///
private IOpen8Code services = null;
///
/// 当前彩种
///
private SCCLottery currentLottery
{
get
{
return SCCLottery.ChongQingKL10F;
}
}
///
/// 邮件接口
///
private IEmail email = null;
#endregion
}
}