using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Quartz;
using SCC.Common;
using SCC.Models;
using SCC.Interface;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace SCC.Crawler.GP
{
///
/// 数据爬取类
/// 北京PK10(北京赛车)
///
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class BJPK10Job : IJob
{
///
/// 构造函数
///
public BJPK10Job()
{
log = new LogHelper();
services = IOC.Resolve();
email = IOC.Resolve();
}
///
/// 作业执行入口
///
/// 作业执行上下文
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
//预设节假日不开奖
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
LatestItem = context.JobDetail.JobDataMap["LatestItem"] as OpenCode10Model;
try
{
//服务启动时配置初始数据
if (LatestItem == null)
{
LatestItem = services.GetLastItem(currentLottery);
}
//第一次启动服务或最新开奖记录为昨日滴,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表
if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
{
CheckingYesterdayTheLotteryData();
LatestItem = GenerateTodayFirstItem();
}
//当最新开奖记录不是今天最后一期,执行当天作业
if (!CheckLatestItemIsTodayLastItem())
{
//var query=services.GetLastItem(currentLottery);
// DoTodayJobByMainUrl();
//DoTodayJobByBackUrl();
DoTodayJobByDB();
}
}
catch (Exception ex)
{
log.Error(typeof(BJPK10Job), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
//保存最新期号和失败期号列表
context.JobDetail.JobDataMap["LatestItem"] = LatestItem;
}
///
/// 新开奖号数据库拿取
///
private void DoYesterdayFailedListByMainUrl_New()
{
var OpenList = GetYesterdayOpenListFromDB_new();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Key == failedQiHao).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(failedQiHao, matchItem.Value, true))
{
//处理成功写入日志
log.Info(typeof(HeBei11X5Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
///
/// 通过数据库点抓取昨日开奖列表
///
///
private Dictionary GetYesterdayOpenListFromDB_new()
{
Dictionary result = new Dictionary();
try
{
var list = services.GetYesterdayFailQQListDB10(currentLottery, Config.TimesPerDay);
result = list;
}
catch (Exception ex)
{
log.Error(typeof(HeBei11X5Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过数据库获取开奖数据
///
private void DoTodayJobByDB()
{
var OpenList = services.GetYesterdayFailQQListDB10_toDay(currentLottery, Config.TimesPerDay);
if (OpenList.Count == 0) return;//无抓取数据
var newestQiHao = OpenList.OrderBy(a => a.Key).Last().Key;
var startQiNum = LatestItem.Term + 1;
var newestQiNum = OpenList.OrderBy(a=>a.Key).Last().Key;
if (Convert.ToInt32(startQiNum) > Convert.ToInt32(newestQiNum)) return;//无最新数据
//处理最新开奖数据
for (var i = startQiNum; i <= Convert.ToInt32(newestQiNum); i++)
{
var matchItem = OpenList.Where(R => R.Key == i.ToString()).FirstOrDefault();
if (matchItem.Key != null && SaveRecord(matchItem.Key, matchItem.Value, true))
{
//处理成功写入日志
log.Info(typeof(BJPK10Job), CommonHelper.GetJobMainLogInfo(Config, i.ToString()));
//LatestItem = matchItem;
}
}
}
///
/// 自检昨天开奖数据
///
private void CheckingYesterdayTheLotteryData()
{
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据
//从数据库中获取昨天数据抓取失败列表
FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, GenerateYesterdayQiHaoList());
if (FailedQiHaoList.Count > 0)
{
DoYesterdayFailedListByMainUrl_New();
//DoYesterdayFailedListByMainUrl();
//DoYesterdayFailedListByBackUrl();
foreach (var fQiHao in FailedQiHaoList)
{
//将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中
if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, GenerateYesterdayPK10OpenTime(fQiHao)))
log.Error(typeof(BJPK10Job), CommonHelper.GetJobLogError(Config, fQiHao));
}
}
}
///
/// 通过主站点抓取开奖数据
/// (百度彩票)
///
private void DoTodayJobByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl))
{
var OpenList = GetTodayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var startQiNum = LatestItem.Term + 1;
var newestQiNum = OpenList.Last().Term;
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
for (var i = startQiNum; i <= newestQiNum; i++)
{
var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJPK10Job), CommonHelper.GetJobMainLogInfo(Config, i.ToString()));
LatestItem = matchItem;
}
}
}
}
///
/// 通过主站抓取错误期号列表中每一个期号
/// (百度彩票)
///
private void DoYesterdayFailedListByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJPK10Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过主站点抓取今日最新开奖数据
///
///
private List GetTodayOpenListFromMainUrl()
{
List result = new List();
try
{
var HtmlResource = NetHelper.GetBaiDuLeCaiResponse(Config.MainUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
Regex reg = new Regex(@"var phaseData = ([\s\S]*?);");
Match m = reg.Match(HtmlResource);
if (!m.Success) return result;
var dataJson = m.Result("$1");
JObject obj = JsonConvert.DeserializeObject(dataJson);
JToken items = obj[CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd")];
if (items == null || !items.HasValues) return result;
OpenCode10Model model = null;
JObject childObject = null, openCode = null;
JArray red = null;
foreach (JProperty item in items)
{
model = new OpenCode10Model();
model.Term = Convert.ToInt64(item.Name);//开奖期号
childObject = (JObject)item.Value;
model.OpenTime = Convert.ToDateTime(childObject["open_time"]);//开奖时间
openCode = (JObject)childObject["result"];
red = (JArray)openCode["red"];//20个开奖球号
if (red.Count != 10) continue;
model.OpenCode1 = Convert.ToInt32(red[0].ToString());
model.OpenCode2 = Convert.ToInt32(red[1].ToString());
model.OpenCode3 = Convert.ToInt32(red[2].ToString());
model.OpenCode4 = Convert.ToInt32(red[3].ToString());
model.OpenCode5 = Convert.ToInt32(red[4].ToString());
model.OpenCode6 = Convert.ToInt32(red[5].ToString());
model.OpenCode7 = Convert.ToInt32(red[6].ToString());
model.OpenCode8 = Convert.ToInt32(red[7].ToString());
model.OpenCode9 = Convert.ToInt32(red[8].ToString());
model.OpenCode10 = Convert.ToInt32(red[9].ToString());
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJPK10Job), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过主站点抓取昨日开奖数据
///
///
private List GetYesterdayOpenListFromMainUrl()
{
List result = new List();
try
{
var requestUrl = string.Format("{0}/{1}", Config.MainUrl, CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd"));
var HtmlResource = NetHelper.GetBaiDuLeCaiResponse(requestUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
Regex reg = new Regex(@"var phaseData = ([\s\S]*?);");
Match m = reg.Match(HtmlResource);
if (!m.Success) return result;
var dataJson = m.Result("$1");
JObject obj = JsonConvert.DeserializeObject(dataJson);
JToken items = obj[CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd")];
if (items == null || !items.HasValues) return result;
OpenCode10Model model = null;
JObject childObject = null, openCode = null;
JArray red = null;
foreach (JProperty item in items)
{
model = new OpenCode10Model();
model.Term = Convert.ToInt64(item.Name);//开奖期号
childObject = (JObject)item.Value;
model.OpenTime = Convert.ToDateTime(childObject["open_time"]);//开奖时间
openCode = (JObject)childObject["result"];
red = (JArray)openCode["red"];//20个开奖球号
if (red.Count != 10) continue;
model.OpenCode1 = Convert.ToInt32(red[0].ToString());
model.OpenCode2 = Convert.ToInt32(red[1].ToString());
model.OpenCode3 = Convert.ToInt32(red[2].ToString());
model.OpenCode4 = Convert.ToInt32(red[3].ToString());
model.OpenCode5 = Convert.ToInt32(red[4].ToString());
model.OpenCode6 = Convert.ToInt32(red[5].ToString());
model.OpenCode7 = Convert.ToInt32(red[6].ToString());
model.OpenCode8 = Convert.ToInt32(red[7].ToString());
model.OpenCode9 = Convert.ToInt32(red[8].ToString());
model.OpenCode10 = Convert.ToInt32(red[9].ToString());
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJPK10Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过备用站点抓取开奖数据
/// (北京福彩官网)
///
private void DoTodayJobByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl))
{
var OpenList = GetOpenListFromBackUrl(Config.BackUrl);
if (OpenList.Count == 0) return;//无抓取数据
foreach (OpenCode10Model model in OpenList)
{
if (services.AddOpen10Code(currentLottery, model))
{
//处理成功写入日志
log.Info(typeof(BJPK10Job), CommonHelper.GetJobBackLogInfo(Config, model.Term.ToString()));
LatestItem = model;
}
}
//var startQiNum = LatestItem.Term + 1;
//var newestQiNum = OpenList.First().Term;
//if (startQiNum > newestQiNum) return;//无最新数据
////处理最新开奖数据
//for (var i = startQiNum; i <= newestQiNum; i++)
//{
// var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
// if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
// {
// //处理成功写入日志
// log.Info(typeof(BJPK10Job), CommonHelper.GetJobBackLogInfo(Config, i.ToString()));
// LatestItem = matchItem;
// }
//}
}
}
///
/// 通过备用地址抓取错误期号列表中每一个期号
/// (北京福彩官网)
///
private void DoYesterdayFailedListByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetOpenListFromBackUrl(Config.BackUrl);
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (var failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJPK10Job), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 抓取备用站点开奖数据
/// 北京福彩官网分页数据太多,只有通过主站点来保证开奖数据完整性
///
/// 备用站点
///
private List GetOpenListFromBackUrl(string url)
{
List result = new List();
try
{
string HtmlResource = string.Empty;
HtmlResource = NetHelper.GetUrlResponse(url);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectNodes("//table");
if (table == null || table.Count < 2) return result;
var trs = table[1].ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
if (trs.Count < 2) return result;
List tds = null;
OpenCode10Model model = null;
string[] openCodeList = null;
string openTime = string.Empty;
string todayDateString = CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd");
for (var i = 1; i < trs.Count; i++)//第一行是表头
{
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 3) continue;
openTime = tds[2].InnerText.Trim();
if (!openTime.StartsWith(todayDateString)) continue;
model = new OpenCode10Model();
model.Term = Convert.ToInt64(tds[0].InnerText.Trim());
openCodeList = tds[1].InnerText.Trim().Split(new char[] { ',' });
if (openCodeList.Length < 10) continue;
model.OpenCode1 = Convert.ToInt32(openCodeList[0]);
model.OpenCode2 = Convert.ToInt32(openCodeList[1]);
model.OpenCode3 = Convert.ToInt32(openCodeList[2]);
model.OpenCode4 = Convert.ToInt32(openCodeList[3]);
model.OpenCode5 = Convert.ToInt32(openCodeList[4]);
model.OpenCode6 = Convert.ToInt32(openCodeList[5]);
model.OpenCode7 = Convert.ToInt32(openCodeList[6]);
model.OpenCode8 = Convert.ToInt32(openCodeList[7]);
model.OpenCode9 = Convert.ToInt32(openCodeList[8]);
model.OpenCode10 = Convert.ToInt32(openCodeList[9]);
model.OpenTime = Convert.ToDateTime(openTime);
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJPK10Job), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 生成今日第一期记录
///
///
private OpenCode10Model GenerateTodayFirstItem()
{
OpenCode10Model item = new OpenCode10Model();
var currentDateTime = CommonHelper.SCCSysDateTime;
TimeSpan datepart = currentDateTime - new DateTime(2018, 1, 1);
var t = Config.SkipDate.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
item.Term = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear;//期号使用昨日最后一期期号,便于计算使用
item.OpenTime = new DateTime(currentDateTime.Year, currentDateTime.Month, currentDateTime.Day, Config.StartHour, Config.StartMinute, 0);
return item;
}
///
/// 核实最新一期开奖记录是否为今日最后一期
///
/// 期号
///
private bool CheckLatestItemIsTodayLastItem()
{
if (LatestItem == null ||
LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
return true;
var firstItem = GenerateTodayFirstItem();
if (LatestItem.Term == (firstItem.Term + Config.TimesPerDay))
return true;
return false;
}
///
/// 获取今天第一期期号
///
///
private string GenerateTodayFirstQiHao()
{
var datepart = CommonHelper.SCCSysDateTime - new DateTime(2018, 1, 1);
var t = Config.SkipDate.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var terms = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear + 1;
return terms.ToString();
}
///
/// 生成昨天的期号列表
///
///
private List GenerateYesterdayQiHaoList()
{
List result = new List();
var todayFirstQiHao = Convert.ToInt64(GenerateTodayFirstQiHao());
for (var i = Config.TimesPerDay; i > 0; i--)
{
result.Add((todayFirstQiHao - i).ToString());
}
return result;
}
///
/// 生成PK10的开奖时间
///
///
///
private DateTime GenerateYesterdayPK10OpenTime(string QiHao)
{
var openDay = CommonHelper.SCCSysDateTime.AddDays(-1);
var StartTime = new DateTime(openDay.Year, openDay.Month, openDay.Day, Config.StartHour, Config.StartMinute, 0);
var t = (Convert.ToInt64(QiHao) - LastTermLastYear) % Config.TimesPerDay;
if (t == 0)
return StartTime.AddMinutes((Config.TimesPerDay - 1) * Config.Interval);
else
return StartTime.AddMinutes((t - 1) * Config.Interval);
}
///
/// 将此彩种指定期号和开奖号码保存至数据库
///
/// 期号
/// 开奖号码(形如01,02,03)
/// 是否是保存昨天的记录
///
private bool SaveRecord(string QiHao, string OpenCode, bool IsYesterdayRecord)
{
if (!string.IsNullOrWhiteSpace(QiHao) && !string.IsNullOrWhiteSpace(OpenCode))
{
var list = OpenCode.Split(',');
OpenCode10Model model = new OpenCode10Model();
model.Term = Convert.ToInt64(QiHao);//开奖期号
model.OpenTime = Convert.ToDateTime(list[10]);
model.OpenCode1 = Convert.ToInt32(list[0].ToString());
model.OpenCode2 = Convert.ToInt32(list[1].ToString());
model.OpenCode3 = Convert.ToInt32(list[2].ToString());
model.OpenCode4 = Convert.ToInt32(list[3].ToString());
model.OpenCode5 = Convert.ToInt32(list[4].ToString());
model.OpenCode6 = Convert.ToInt32(list[5].ToString());
model.OpenCode7 = Convert.ToInt32(list[6].ToString());
model.OpenCode8 = Convert.ToInt32(list[7].ToString());
model.OpenCode9 = Convert.ToInt32(list[8].ToString());
model.OpenCode10 = Convert.ToInt32(list[9].ToString());
return services.AddOpen10Code(currentLottery, model);
}
return false;
}
#region Attribute
///
/// 配置信息
///
private SCCConfig Config = null;
///
/// 当天抓取的最新一期开奖记录
///
private OpenCode10Model LatestItem = null;
///
/// 当天抓取失败列表
///
private List FailedQiHaoList = null;
///
/// 日志对象
///
private LogHelper log = null;
///
/// 数据服务
///
private IOpen10Code services = null;
///
/// 当前彩种
///
private SCCLottery currentLottery
{
get
{
return SCCLottery.BeiJingPK10;
}
}
///
/// 邮件接口
///
private IEmail email = null;
///
/// 2016年最后一期期号
///
private long LastTermLastYear = 659223;
#endregion
}
}