using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Quartz;
using SCC.Common;
using SCC.Models;
using SCC.Interface;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace SCC.Crawler.GP
{
///
/// 数据爬取类
/// 吉林快3
///
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class JLK3Job : IJob
{
///
/// 构造函数
///
public JLK3Job()
{
log = new LogHelper();
services = IOC.Resolve();
email = IOC.Resolve();
}
///
/// 作业执行入口
///
/// 作业执行上下文
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
//预设节假日不开奖
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
LatestQiHao = context.JobDetail.JobDataMap.GetString("LatestQiHao");
try
{
//服务启动时配置初始数据
if (string.IsNullOrEmpty(LatestQiHao))
{
var lastItem = services.GetLastItem(currentLottery);
if (lastItem != null)
{
LatestQiHao = lastItem.Term.ToString();
}
}
//第一次启动服务或最新期号为昨天的开奖期号,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表
if (string.IsNullOrEmpty(LatestQiHao) || !LatestQiHao.StartsWith(CommonHelper.SCCSysDateTime.ToString("yyMMdd")))
{
CheckingYesterdayTheLotteryData();
LatestQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(0);
}
//当最新期号不符合当天总期数,执行当天作业
if (Convert.ToInt32(LatestQiHao.Substring(6)) != Config.TimesPerDay)
{
DoTodayJobByMainUrl();
DoTodayJobByBackUrl();
}
}
catch (Exception ex)
{
log.Error(typeof(JLK3Job), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
//保存最新期号和失败期号列表
context.JobDetail.JobDataMap["LatestQiHao"] = LatestQiHao;
}
///
/// 自检昨天开奖数据
///
private void CheckingYesterdayTheLotteryData()
{
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据
//从数据库中获取昨天数据抓取失败列表
FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, Config.TimesPerDay);
if (FailedQiHaoList.Count > 0)
{
DoYesterdayFailedListByMainUrl();
DoYesterdayFailedListByBackUrl();
foreach (var fQiHao in FailedQiHaoList)
{
//将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中
if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, CommonHelper.GenerateYesterdayOpenTime(Config, fQiHao)))
log.Error(typeof(JLK3Job), CommonHelper.GetJobLogError(Config, fQiHao));
}
}
}
///
/// 通过主站点抓取开奖数据
/// (爱彩乐)
///
private void DoTodayJobByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl))
{
var OpenList = GetTodayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.First().Key;
var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1;
var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6));
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
string getQiHao = string.Empty;
for (var i = startQiNum; i <= newestQiNum; i++)
{
getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i);
var matchItem = OpenList.Where(R => R.Key == getQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(getQiHao, matchItem.Value, false))
{
//处理成功写入日志
log.Info(typeof(JLK3Job), CommonHelper.GetJobMainLogInfo(Config, getQiHao));
LatestQiHao = getQiHao;
}
}
}
}
///
/// 通过主站抓取错误期号列表中每一个期号
/// (爱彩乐)
///
private void DoYesterdayFailedListByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true))
{
//处理成功写入日志
log.Info(typeof(JLK3Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过主站点抓取今日最新开奖列表
///
///
private Dictionary GetTodayOpenListFromMainUrl()
{
Dictionary result = new Dictionary();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectSingleNode("//table");
if (table == null) return result;
var trs = table.ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
List tds = null, ems = null;
var matchQiHao = string.Empty;
var matchKJHaoMa = string.Empty;
for (var i = 1; i < trs.Count; i++)//第一行为表头
{
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 3) continue;
matchQiHao = tds[0].InnerText.Trim();
ems = tds[2].ChildNodes.Where(R => R.Name.ToLower() == "em").ToList();
if (ems.Count < 3) continue;
matchKJHaoMa = string.Format("{0},{1},{2}", ems[0].InnerText.Trim(), ems[1].InnerText.Trim(), ems[2].InnerText.Trim());
if (!result.ContainsKey(matchQiHao))
result.Add(matchQiHao, matchKJHaoMa);
}
}
}
catch (Exception ex)
{
log.Error(typeof(JLK3Job), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过主站点抓取昨日开奖列表
///
///
private Dictionary GetYesterdayOpenListFromMainUrl()
{
Dictionary result = new Dictionary();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl + "?action=chart&date=yesterday&id=806&async=true");
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
var obj = JsonConvert.DeserializeObject(HtmlResource);
if (obj != null && obj["data"] != null)
{
var matchQiHao = string.Empty;
var matchKJHaoMa = string.Empty;
JArray openCodeList = null;
foreach (var item in obj["data"])
{
matchQiHao = item["dateNumber"].ToString();
openCodeList = (JArray)item["list"];
matchKJHaoMa = string.Format("{0},{1},{2}", openCodeList[0].ToString(), openCodeList[1].ToString(), openCodeList[2].ToString());
if (!result.ContainsKey(matchQiHao))
result.Add(matchQiHao, matchKJHaoMa);
}
}
}
}
catch (Exception ex)
{
log.Error(typeof(JLK3Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过备用站点抓取开奖数据
/// (彩乐乐)
///
private void DoTodayJobByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl))
{
var OpenList = GetTodayOpenListFromBackUrl();
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.Last().Key;
var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1;
var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6));
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
var getQiHao = string.Empty;
for (var i = startQiNum; i <= newestQiNum; i++)
{
getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQQ(i);
var matchItem = OpenList.Where(R => R.Key == getQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(getQiHao, matchItem.Value, false))
{
//处理成功写入日志
log.Info(typeof(JLK3Job), CommonHelper.GetJobBackLogInfo(Config, getQiHao));
LatestQiHao = getQiHao;
}
}
}
}
///
/// 通过备用地址抓取错误期号列表中每一个期号
/// (彩乐乐)
///
private void DoYesterdayFailedListByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromBackUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (var failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true))
{
//处理成功写入日志
log.Info(typeof(JLK3Job), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过备用站点抓取今日最新开奖数据
///
///
private Dictionary GetTodayOpenListFromBackUrl()
{
var requestUrl = string.Format(Config.BackUrl, 0);
return GetOpenListFromBackUrl(requestUrl);
}
///
/// 通过备用站点抓取昨日开奖数据
///
///
private Dictionary GetYesterdayOpenListFromBackUrl()
{
var requestUrl = string.Format(Config.BackUrl, 1);
return GetOpenListFromBackUrl(requestUrl);
}
///
/// 通过备用站点抓取开奖数据
///
/// 备用站点
///
private Dictionary GetOpenListFromBackUrl(string requestUrl)
{
Dictionary result = new Dictionary();
try
{
var HtmlResource = NetHelper.GetUrlResponse(requestUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var tbody = doc.GetElementbyId("cpdata_new");
if (tbody == null) return result;
var trs = tbody.ChildNodes.Where(node => node.Name == "tr").ToList();
string matchQiHao = string.Empty;
string matchKJHaoMa = string.Empty;
for (var i = 0; i < trs.Count; i++)//第一行是表头
{
var trstyle = trs[i].Attributes["style"];
if (trstyle != null && trstyle.Value == "display:none")
{
continue;
}
var tds = trs[i].ChildNodes.Where(node => node.Name == "td").ToList();
matchQiHao = tds[0].InnerText.Trim();
matchKJHaoMa = string.Format("{0},{1},{2}", tds[1].InnerText.Trim(), tds[2].InnerText.Trim(), tds[3].InnerText.Trim());
result.Add(matchQiHao, matchKJHaoMa);
}
}
}
catch (Exception ex)
{
log.Error(typeof(JLK3Job), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 将此彩种指定期号和开奖号码保存至数据库
///
/// 期号
/// 开奖号码(形如01,02,03)
/// 是否是保存昨天的记录
///
private bool SaveRecord(string QiHao, string OpenCode, bool IsYesterdayRecord)
{
if (!string.IsNullOrWhiteSpace(QiHao) && !string.IsNullOrWhiteSpace(OpenCode))
{
OpenCode3Model model = new OpenCode3Model();
model.Term = Convert.ToInt64(QiHao);
var haoMaArray = OpenCode.Split(new char[] { ',' });
model.OpenCode1 = Convert.ToInt32(haoMaArray[0]);
model.OpenCode2 = Convert.ToInt32(haoMaArray[1]);
model.OpenCode3 = Convert.ToInt32(haoMaArray[2]);
if (IsYesterdayRecord)
model.OpenTime = CommonHelper.GenerateYesterdayOpenTime(Config, QiHao);
else
model.OpenTime = CommonHelper.GenerateTodayOpenTime(Config, QiHao);
return services.AddOpen3Code(currentLottery, model);
}
return false;
}
#region Attribute
///
/// 配置信息
///
private SCCConfig Config = null;
///
/// 当天抓取的最新一期期号
///
private string LatestQiHao = null;
///
/// 当天抓取失败列表
///
private List FailedQiHaoList = null;
///
/// 日志对象
///
private LogHelper log = null;
///
/// 数据服务
///
private IOpen3Code services = null;
///
/// 当前彩种
///
private SCCLottery currentLottery
{
get
{
return SCCLottery.JiLinK3;
}
}
///
/// 邮件接口
///
private IEmail email = null;
#endregion
}
}