using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;
using Quartz;
using SCC.Common;
using SCC.Models;
using SCC.Interface;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace SCC.Crawler.GP
{
///
/// 数据爬取类
/// 北京快乐8
///
[DisallowConcurrentExecution]
[PersistJobDataAfterExecution]
public class BJKL8Job : IJob
{
///
/// 构造函数
///
public BJKL8Job()
{
log = new LogHelper();
services = IOC.Resolve();
email = IOC.Resolve();
}
///
/// 作业执行入口
///
/// 作业执行上下文
public void Execute(IJobExecutionContext context)
{
Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
//预设节假日不开奖
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
LatestItem = context.JobDetail.JobDataMap["LatestItem"] as OpenCode21Model;
try
{
//服务启动时配置初始数据
if (LatestItem == null)
{
LatestItem = services.GetLastItem(currentLottery);
}
//第一次启动服务或最新开奖记录为昨日滴,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表
if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
{
CheckingYesterdayTheLotteryData();
LatestItem = GenerateTodayFirstItem();
}
//当最新开奖记录不是今天最后一期,执行当天作业
if (!CheckLatestItemIsTodayLastItem())
{
DoTodayJobByMainUrl();
//DoTodayJobByBackUrl();
}
}
catch (Exception ex)
{
log.Error(typeof(BJKL8Job), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
//保存最新期号和失败期号列表
context.JobDetail.JobDataMap["LatestItem"] = LatestItem;
}
///
/// 自检昨天开奖数据
///
private void CheckingYesterdayTheLotteryData()
{
if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd"))) return;//如果昨日设定不开奖则不自检昨日开奖数据
//从数据库中获取昨天数据抓取失败列表
FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, GenerateYesterdayQiHaoList());
if (FailedQiHaoList.Count > 0)
{
GetYesterdayOpenListFromMainUrl_New();
//DoYesterdayFailedListByMainUrl();
//DoYesterdayFailedListByBackUrl();
foreach (var fQiHao in FailedQiHaoList)
{
//将抓取失败数据推送至邮件列表,待邮件服务发送至配置管理员的邮箱中
if (email.AddEmail(Config.Area + Config.LotteryName, fQiHao, GenerateYesterdayKL8OpenTime(fQiHao)))
log.Error(typeof(BJKL8Job), CommonHelper.GetJobLogError(Config, fQiHao));
}
}
}
///
/// 通过主站点抓取开奖数据
/// (百度彩票)
///
private void DoTodayJobByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl))
{
var OpenList = GetTodayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var startQiNum = LatestItem.Term;
var newestQiNum = OpenList.First().Term;
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
for (var i = startQiNum; i <= newestQiNum; i++)
{
var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJKL8Job), CommonHelper.GetJobMainLogInfo(Config, i.ToString()));
LatestItem = matchItem;
}
}
}
}
///
/// 通过主站抓取错误期号列表中每一个期号
/// (百度彩票)
///
private void DoYesterdayFailedListByMainUrl()
{
if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetYesterdayOpenListFromMainUrl();
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (string failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJKL8Job), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
///
/// 通过主站点抓取今日最新开奖数据
///
///
private List GetTodayOpenListFromMainUrl()
{
List result = new List();
try
{
var requestUrl = string.Format(Config.MainUrl, DateTime.Now.ToString("yyyy-MM-dd"));
var HtmlResource = NetHelper.GetUrlResponse(requestUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
OpenCode21Model model = null;
JObject obj = JsonConvert.DeserializeObject(HtmlResource);
var data = obj["data"];
var list = JsonConvert.DeserializeObject>(data.ToString());
foreach (var item in list)
{
var red = item.opencode.Split(',');
var extra = red[19].Split('+');
red[19] = extra[0];
model = new OpenCode21Model();
model.Term = Convert.ToInt64(item.expect);//开奖期号
model.OpenTime = Convert.ToDateTime(item.opentime);//开奖时间
model.OpenCode21 = Convert.ToInt32(extra[1].ToString());//飞盘
model.OpenCode1 = Convert.ToInt32(red[0].ToString());
model.OpenCode2 = Convert.ToInt32(red[1].ToString());
model.OpenCode3 = Convert.ToInt32(red[2].ToString());
model.OpenCode4 = Convert.ToInt32(red[3].ToString());
model.OpenCode5 = Convert.ToInt32(red[4].ToString());
model.OpenCode6 = Convert.ToInt32(red[5].ToString());
model.OpenCode7 = Convert.ToInt32(red[6].ToString());
model.OpenCode8 = Convert.ToInt32(red[7].ToString());
model.OpenCode9 = Convert.ToInt32(red[8].ToString());
model.OpenCode10 = Convert.ToInt32(red[9].ToString());
model.OpenCode11 = Convert.ToInt32(red[10].ToString());
model.OpenCode12 = Convert.ToInt32(red[11].ToString());
model.OpenCode13 = Convert.ToInt32(red[12].ToString());
model.OpenCode14 = Convert.ToInt32(red[13].ToString());
model.OpenCode15 = Convert.ToInt32(red[14].ToString());
model.OpenCode16 = Convert.ToInt32(red[15].ToString());
model.OpenCode17 = Convert.ToInt32(red[16].ToString());
model.OpenCode18 = Convert.ToInt32(red[17].ToString());
model.OpenCode19 = Convert.ToInt32(red[18].ToString());
model.OpenCode20 = Convert.ToInt32(red[19].ToString());
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJKL8Job), string.Format("【{0}】通过主站点抓取今日最新开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过主站点抓取昨日开奖数据
///
///
private List GetYesterdayOpenListFromMainUrl()
{
List result = new List();
try
{
var requestUrl = string.Format("{0}/{1}", Config.MainUrl, CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd"));
var HtmlResource = NetHelper.GetBaiDuLeCaiResponse(requestUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
Regex reg = new Regex(@"var phaseData = ([\s\S]*?);");
Match m = reg.Match(HtmlResource);
if (!m.Success) return result;
var dataJson = m.Result("$1");
JObject obj = JsonConvert.DeserializeObject(dataJson);
JToken items = obj[CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyy-MM-dd")];
if (items == null || !items.HasValues) return result;
OpenCode21Model model = null;
JObject childObject = null, openCode = null;
JArray extra = null, red = null;
foreach (JProperty item in items)
{
model = new OpenCode21Model();
model.Term = Convert.ToInt64(item.Name);//开奖期号
childObject = (JObject)item.Value;
model.OpenTime = Convert.ToDateTime(childObject["open_time"]);//开奖时间
openCode = (JObject)childObject["result"];
extra = (JArray)openCode["extra"];
if (extra.Count != 1) continue;
model.OpenCode21 = Convert.ToInt32(extra[0].ToString());//飞盘
red = (JArray)openCode["red"];//20个开奖球号
if (red.Count != 20) continue;
model.OpenCode1 = Convert.ToInt32(red[0].ToString());
model.OpenCode2 = Convert.ToInt32(red[1].ToString());
model.OpenCode3 = Convert.ToInt32(red[2].ToString());
model.OpenCode4 = Convert.ToInt32(red[3].ToString());
model.OpenCode5 = Convert.ToInt32(red[4].ToString());
model.OpenCode6 = Convert.ToInt32(red[5].ToString());
model.OpenCode7 = Convert.ToInt32(red[6].ToString());
model.OpenCode8 = Convert.ToInt32(red[7].ToString());
model.OpenCode9 = Convert.ToInt32(red[8].ToString());
model.OpenCode10 = Convert.ToInt32(red[9].ToString());
model.OpenCode11 = Convert.ToInt32(red[10].ToString());
model.OpenCode12 = Convert.ToInt32(red[11].ToString());
model.OpenCode13 = Convert.ToInt32(red[12].ToString());
model.OpenCode14 = Convert.ToInt32(red[13].ToString());
model.OpenCode15 = Convert.ToInt32(red[14].ToString());
model.OpenCode16 = Convert.ToInt32(red[15].ToString());
model.OpenCode17 = Convert.ToInt32(red[16].ToString());
model.OpenCode18 = Convert.ToInt32(red[17].ToString());
model.OpenCode19 = Convert.ToInt32(red[18].ToString());
model.OpenCode20 = Convert.ToInt32(red[19].ToString());
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJKL8Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 通过主站点抓取昨日开奖数据
///
///
private List GetYesterdayOpenListFromMainUrl_New()
{
List result = new List();
try
{
var requestUrl = string.Format(Config.MainUrl,DateTime.Now.AddDays(-1).ToString("yyyy-MM-dd"));
var HtmlResource = NetHelper.GetUrlResponse(requestUrl);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
OpenCode21Model model = null;
JObject obj = JsonConvert.DeserializeObject(HtmlResource);
var data = obj["data"];
var SuccessList = new List();
var list = JsonConvert.DeserializeObject>(data.ToString());
foreach (var item in list)
{
var red = item.opencode.Split(',');
var extra = red[19].Split('+');
red[19] = extra[0];
model = new OpenCode21Model();
model.Term = Convert.ToInt64(item.expect);//开奖期号
model.OpenTime = Convert.ToDateTime(item.opentime);//开奖时间
model.OpenCode21 = Convert.ToInt32(extra[1].ToString());//飞盘
model.OpenCode1 = Convert.ToInt32(red[0].ToString());
model.OpenCode2 = Convert.ToInt32(red[1].ToString());
model.OpenCode3 = Convert.ToInt32(red[2].ToString());
model.OpenCode4 = Convert.ToInt32(red[3].ToString());
model.OpenCode5 = Convert.ToInt32(red[4].ToString());
model.OpenCode6 = Convert.ToInt32(red[5].ToString());
model.OpenCode7 = Convert.ToInt32(red[6].ToString());
model.OpenCode8 = Convert.ToInt32(red[7].ToString());
model.OpenCode9 = Convert.ToInt32(red[8].ToString());
model.OpenCode10 = Convert.ToInt32(red[9].ToString());
model.OpenCode11 = Convert.ToInt32(red[10].ToString());
model.OpenCode12 = Convert.ToInt32(red[11].ToString());
model.OpenCode13 = Convert.ToInt32(red[12].ToString());
model.OpenCode14 = Convert.ToInt32(red[13].ToString());
model.OpenCode15 = Convert.ToInt32(red[14].ToString());
model.OpenCode16 = Convert.ToInt32(red[15].ToString());
model.OpenCode17 = Convert.ToInt32(red[16].ToString());
model.OpenCode18 = Convert.ToInt32(red[17].ToString());
model.OpenCode19 = Convert.ToInt32(red[18].ToString());
model.OpenCode20 = Convert.ToInt32(red[19].ToString());
if (item.expect != null && services.AddOpen21Code(currentLottery, model))
{
//处理成功写入日志
log.Info(typeof(BJKL8Job), CommonHelper.GetJobMainLogInfo(Config, item.expect));
SuccessList.Add(item.expect);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJKL8Job), string.Format("【{0}】通过主站点抓取昨日开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
public class bjkl8Model {
public string expect { get; set; }
public string opencode { get; set; }
public string opentime { get; set; }
}
///
/// 通过备用站点抓取开奖数据
/// (北京福彩官网)
///
private void DoTodayJobByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl))
{
var OpenList = GetOpenListFromBackUrl(Config.BackUrl);
if (OpenList.Count == 0) return;//无抓取数据
var startQiNum = LatestItem.Term + 1;
var newestQiNum = OpenList.First().Term;
if (startQiNum > newestQiNum) return;//无最新数据
//处理最新开奖数据
for (var i = startQiNum; i <= newestQiNum; i++)
{
var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJKL8Job), CommonHelper.GetJobBackLogInfo(Config, i.ToString()));
LatestItem = matchItem;
}
}
}
}
///
/// 通过备用地址抓取错误期号列表中每一个期号
/// (北京福彩官网)
///
private void DoYesterdayFailedListByBackUrl()
{
if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
{
var OpenList = GetOpenListFromBackUrl(Config.BackUrl);
if (OpenList.Count == 0) return;//无抓取数据
var SuccessList = new List();
foreach (var failedQiHao in FailedQiHaoList)
{
var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
if (matchItem != null && services.AddOpen21Code(currentLottery, matchItem))
{
//处理成功写入日志
log.Info(typeof(BJKL8Job), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
SuccessList.Add(failedQiHao);
continue;
}
}
foreach (var successQiHao in SuccessList)
{
FailedQiHaoList.Remove(successQiHao);
}
//foreach (OpenCode21Model model in OpenList)
//{
// if (services.AddOpen21Code(currentLottery, model))
// {
// //处理成功写入日志
// log.Info(typeof(BJKL8Job), CommonHelper.GetJobBackLogInfo(Config, model.Term.ToString()));
// }
//}
}
}
///
/// 抓取备用站点开奖数据
/// 由于北京福彩官网分页30条数据一页,而此彩种每天开179期,如果去跑6页,负荷太大,只有通过主站点来保证数据完整性
///
/// 备用站点
///
private List GetOpenListFromBackUrl(string url)
{
List result = new List();
try
{
string HtmlResource = string.Empty;
HtmlResource = NetHelper.GetUrlResponse(url);
if (!string.IsNullOrWhiteSpace(HtmlResource))
{
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(HtmlResource);
var table = doc.DocumentNode.SelectNodes("//table");
if (table == null || table.Count < 2) return result;
var trs = table[1].ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
if (trs.Count < 2) return result;
List tds = null;
OpenCode21Model model = null;
string[] openCodeList = null;
string openTime = string.Empty;
string todayDateString = CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd");
for (var i = 1; i < trs.Count; i++)//第一行是表头
{
tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
if (tds.Count < 4) continue;
openTime = tds[3].InnerText.Trim();
if (!openTime.StartsWith(todayDateString)) continue;
model = new OpenCode21Model();
model.Term = Convert.ToInt64(tds[0].InnerText.Trim());
model.OpenCode21 = Convert.ToInt32(tds[2].InnerText.Trim());//飞盘
openCodeList = tds[1].InnerText.Trim().Split(new char[] { ',' });
if (openCodeList.Length < 20) continue;
model.OpenCode1 = Convert.ToInt32(openCodeList[0]);
model.OpenCode2 = Convert.ToInt32(openCodeList[1]);
model.OpenCode3 = Convert.ToInt32(openCodeList[2]);
model.OpenCode4 = Convert.ToInt32(openCodeList[3]);
model.OpenCode5 = Convert.ToInt32(openCodeList[4]);
model.OpenCode6 = Convert.ToInt32(openCodeList[5]);
model.OpenCode7 = Convert.ToInt32(openCodeList[6]);
model.OpenCode8 = Convert.ToInt32(openCodeList[7]);
model.OpenCode9 = Convert.ToInt32(openCodeList[8]);
model.OpenCode10 = Convert.ToInt32(openCodeList[9]);
model.OpenCode11 = Convert.ToInt32(openCodeList[10]);
model.OpenCode12 = Convert.ToInt32(openCodeList[11]);
model.OpenCode13 = Convert.ToInt32(openCodeList[12]);
model.OpenCode14 = Convert.ToInt32(openCodeList[13]);
model.OpenCode15 = Convert.ToInt32(openCodeList[14]);
model.OpenCode16 = Convert.ToInt32(openCodeList[15]);
model.OpenCode17 = Convert.ToInt32(openCodeList[16]);
model.OpenCode18 = Convert.ToInt32(openCodeList[17]);
model.OpenCode19 = Convert.ToInt32(openCodeList[18]);
model.OpenCode20 = Convert.ToInt32(openCodeList[19]);
model.OpenTime = Convert.ToDateTime(openTime);
if (!result.Contains(model))
result.Add(model);
}
}
}
catch (Exception ex)
{
log.Error(typeof(BJKL8Job), string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + Config.LotteryName, ex.Message));
}
return result;
}
///
/// 生成今日第一期记录
///
///
private OpenCode21Model GenerateTodayFirstItem()
{
OpenCode21Model item = new OpenCode21Model();
var currentDateTime = CommonHelper.SCCSysDateTime;
TimeSpan datepart = currentDateTime - new DateTime(2017, 1, 1);
var t = Config.SkipDate.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
item.Term = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear-6;//期号使用昨日最后一期期号,便于计算使用
item.OpenTime = new DateTime(currentDateTime.Year, currentDateTime.Month, currentDateTime.Day, Config.StartHour, Config.StartMinute, 0);
return item;
}
///
/// 核实最新一期开奖记录是否为今日最后一期
///
/// 期号
///
private bool CheckLatestItemIsTodayLastItem()
{
if (LatestItem == null ||
LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
return true;
var firstItem = GenerateTodayFirstItem();
if (LatestItem.Term == (firstItem.Term + Config.TimesPerDay))
return true;
return false;
}
///
/// 获取今天第一期期号
///
///
private string GenerateTodayFirstQiHao()
{
TimeSpan datepart = CommonHelper.SCCSysDateTime - new DateTime(2017, 1, 1);
var t = Config.SkipDate.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
var terms = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear + 1;
return terms.ToString();
}
///
/// 生成昨天的期号列表
///
///
private List GenerateYesterdayQiHaoList()
{
List result = new List();
var todayFirstQiHao = Convert.ToInt64(GenerateTodayFirstQiHao());
for (var i = Config.TimesPerDay; i > 0; i--)
{
result.Add((todayFirstQiHao - i).ToString());
}
return result;
}
///
/// 生成快乐8的开奖时间
///
///
///
private DateTime GenerateYesterdayKL8OpenTime(string QiHao)
{
var openDay = CommonHelper.SCCSysDateTime.AddDays(-1);
var StartTime = new DateTime(openDay.Year, openDay.Month, openDay.Day, Config.StartHour, Config.StartMinute, 0);
var t = (Convert.ToInt64(QiHao) - LastTermLastYear) % Config.TimesPerDay;
if (t == 0)
return StartTime.AddMinutes((Config.TimesPerDay - 1) * Config.Interval);
else
return StartTime.AddMinutes((t - 1) * Config.Interval);
}
#region Attribute
///
/// 配置信息
///
private SCCConfig Config = null;
///
/// 当天抓取的最新一期开奖记录
///
private OpenCode21Model LatestItem = null;
///
/// 当天抓取失败列表
///
private List FailedQiHaoList = null;
///
/// 日志对象
///
private LogHelper log = null;
///
/// 数据服务
///
private IOpen21Code services = null;
///
/// 当前彩种
///
private SCCLottery currentLottery
{
get
{
return SCCLottery.BeiJingKL8;
}
}
///
/// 邮件接口
///
private IEmail email = null;
///
/// 2016年最后一期期号
///
private long LastTermLastYear = 800589;
#endregion
}
}