using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml;
using Quartz;
using SCC.Common;
using SCC.Models;
using SCC.Interface;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace SCC.Crawler.GP
{
///
/// 数据爬取类
/// 湖南幸运赛车
///
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class HUNXYSCJob : IJob
{
///
/// 构造函数
///
public HUNXYSCJob()
{
log = new LogHelper();
services = IOC.Resolve();
email = IOC.Resolve();
}
///
/// 作业执行入口
///
/// 作业执行上下文
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
//预设节假日不开奖
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
LatestQiHao = context.JobDetail.JobDataMap.GetString("LatestQiHao");
try
{
//服务启动时配置初始数据
if (string.IsNullOrEmpty(LatestQiHao))
{
var lastItem = services.GetLastItem(currentLottery);
if (lastItem != null)
{
LatestQiHao = lastItem.Term.ToString();
}
}
//第一次启动服务或最新期号为昨天的开奖期号,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表
if (string.IsNullOrEmpty(LatestQiHao) || !LatestQiHao.StartsWith(CommonHelper.SCCSysDateTime.ToString("yyMMdd")))
{
CheckingYesterdayTheLotteryData();
LatestQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQ(0);
}
//当最新期号不符合当天总期数,执行当天作业
if (Convert.ToInt32(LatestQiHao.Substring(6)) != Config.TimesPerDay)
{
DoTodayJobByMainUrl();
DoTodayJobByBackUrl();
}
}
catch (Exception ex)
{
log.Error(typeof(HUNXYSCJob), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
//保存最新期号和失败期号列表
context.JobDetail.JobDataMap["LatestQiHao"] = LatestQiHao;
}
///
/// 自检昨天开奖数据
///
private void CheckingYesterdayTheLotteryData()
{
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据
//从数据库中获取昨天数据抓取失败列表
FailedQiHaoList = services.GetYesterdayFailQQList(currentLottery, Config.TimesPerDay);
if (FailedQiHaoList.Count > 0)
{
DoYesterdayFailedListByMainUrl();
DoYesterdayFailedListByBackUrl();
foreach (var fQiHao in FailedQiHaoList)
{
//将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中
if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, CommonHelper.GenerateYesterdayOpenTime(Config, fQiHao)))
log.Error(typeof(HUNXYSCJob), CommonHelper.GetJobLogError(Config, fQiHao));
}
}
}
///
/// 通过主站点抓取开奖数据
/// (湖南体彩网)
///
private void DoTodayJobByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl))
{
var OpenList = GetOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.First().Key;
var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1;
var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6));
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
string getQiHao = string.Empty;
for (var i = startQiNum; i <= newestQiNum; i++)
{
getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQ(i);
var matchItem = OpenList.Where(R => R.Key == getQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(getQiHao, matchItem.Value, false))
{
//处理成功写入日志
log.Info(typeof(HUNXYSCJob), CommonHelper.GetJobMainLogInfo(Config, getQiHao));
LatestQiHao = getQiHao;
}
}
}
}
///
/// 通过主站抓取错误期号列表中每一个期号
/// (湖南体彩网)
///
private void DoYesterdayFailedListByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true))
{
//处理成功写入日志
log.Info(typeof(HUNXYSCJob), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 抓取主站点开奖数据
///
///
private Dictionary GetOpenListFromMainUrl()
{
Dictionary result = new Dictionary();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.MainUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectSingleNode("//table");
if (table == null) return result;
var trs = table.ChildNodes.Where(node => node.Name == "tr").ToList();
string matchQiHao = string.Empty;
string matchKJHaoMa = string.Empty;
for (var i = 2; i < trs.Count; i++)//第一行为表头
{
var tds = trs[i].ChildNodes.Where(node => node.Name == "td").ToList();
if (tds.Count <= 1) continue;
matchQiHao = tds[0].InnerText.Trim();
matchKJHaoMa = string.Format("{0},{1},{2}", tds[2].InnerText.Trim(), tds[3].InnerText.Trim(), tds[4].InnerText.Trim());
if (!result.ContainsKey(matchQiHao))
result.Add(matchQiHao, matchKJHaoMa);
}
}
}
catch (Exception ex)
{
log.Error(typeof(HUNXYSCJob), string.Format("【{0}】通过主站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过备用站点抓取开奖数据
/// (新浪爱彩)
///
private void DoTodayJobByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl))
{
var OpenList = GetTodayOpenListFromBackUrl();
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.First().Key;
var startQiNum = Convert.ToInt32(LatestQiHao.Substring(6)) + 1;
var newestQiNum = Convert.ToInt32(newestQiHao.Substring(6));
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
var getQiHao = string.Empty;
for (var i = startQiNum; i <= newestQiNum; i++)
{
getQiHao = CommonHelper.GenerateTodayQiHaoYYMMDDQQ(i);
var matchItem = OpenList.Where(R => R.Key == getQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(getQiHao, matchItem.Value, false))
{
//处理成功写入日志
log.Info(typeof(HUNXYSCJob), CommonHelper.GetJobBackLogInfo(Config, getQiHao));
LatestQiHao = getQiHao;
}
}
}
}
///
/// 通过备用地址抓取错误期号列表中每一个期号
/// (新浪爱彩)
///
private void DoYesterdayFailedListByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromBackUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (var failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true))
{
//处理成功写入日志
log.Info(typeof(HUNXYSCJob), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过备用站点抓取今日最新开奖数据
///
///
private Dictionary GetTodayOpenListFromBackUrl()
{
var postData = string.Format("gameIndex=313&searchDate={0}&gameFrom=", CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd"));
return GetOpenListFromBackUrl(postData);
}
///
/// 通过备用站点抓取昨日开奖数据
///
///
private Dictionary GetYesterdayOpenListFromBackUrl()
{
var postData = string.Format("gameIndex=313&searchDate={0}&gameFrom=", CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd"));
return GetOpenListFromBackUrl(postData);
}
///
/// 抓取备用站点开奖数据
///
/// 发送数据
///
private Dictionary GetOpenListFromBackUrl(string PostData)
{
Dictionary result = new Dictionary();
try
{
var HtmlResource = NetHelper.GetUrlResponse(Config.BackUrl, "POST", PostData);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
JObject obj = JsonConvert.DeserializeObject(HtmlResource);
if (obj == null || obj["resultHtml"] == null) return result;
var trString = obj["resultHtml"].ToString();
if (string.IsNullOrWhiteSpace(trString)) return result;
var tableString = string.Format("", trString.Replace("", "
").Replace("
", "
").Substring(5) + "
");
if (string.IsNullOrWhiteSpace(tableString)) return result;
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(tableString);
var table = doc.DocumentNode.SelectSingleNode("//table");
if (table == null) return result;
var trs = table.ChildNodes.Where(node => node.Name == "tr").ToList();
string matchQiHao = string.Empty;
string matchKJHaoMa = string.Empty;
for (var i = 0; i < trs.Count; i++)
{
var tds = trs[i].ChildNodes.Where(node => node.Name == "td").ToList();
if (tds.Count < 3) continue;
matchQiHao = tds[0].InnerText.Replace("-", string.Empty).Replace("期", string.Empty).Trim().Substring(2);
matchKJHaoMa = tds[2].InnerText.Trim();
if (!result.ContainsKey(matchQiHao))
result.Add(matchQiHao, matchKJHaoMa);
}
}
}
catch (Exception ex)
{
log.Error(typeof(HUNXYSCJob), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 将此彩种指定期号和开奖号码保存至数据库
///
/// 期号
/// 开奖号码(形如01,02,03,04,05)
/// 是否是保存昨天的记录
///
private bool SaveRecord(string QiHao, string OpenCode, bool IsYesterdayRecord)
{
if (!string.IsNullOrWhiteSpace(QiHao) && !string.IsNullOrWhiteSpace(OpenCode))
{
OpenCode3Model model = new OpenCode3Model();
model.Term = Convert.ToInt64(QiHao);
var haoMaArray = OpenCode.Split(new char[] { ',' });
model.OpenCode1 = Convert.ToInt32(haoMaArray[0]);
model.OpenCode2 = Convert.ToInt32(haoMaArray[1]);
model.OpenCode3 = Convert.ToInt32(haoMaArray[2]);
if (IsYesterdayRecord)
model.OpenTime = CommonHelper.GenerateYesterdayOpenTime(Config, QiHao);
else
model.OpenTime = CommonHelper.GenerateTodayOpenTime(Config, QiHao);
return services.AddOpen3Code(currentLottery, model);
}
return false;
}
#region Attribute
///
/// 配置信息
///
private SCCConfig Config = null;
///
/// 当天抓取的最新一期期号
///
private string LatestQiHao = null;
///
/// 当天抓取失败列表
///
private List FailedQiHaoList = null;
///
/// 日志对象
///
private LogHelper log = null;
///
/// 数据服务
///
private IOpen3Code services = null;
///
/// 当前彩种
///
private SCCLottery currentLottery
{
get
{
return SCCLottery.HuNanXYSC;
}
}
///
/// 邮件接口
///
private IEmail email = null;
#endregion
}
}