BJPK10Job.cs 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507
  1. using System;
  2. using System.Collections.Generic;
  3. using System.Linq;
  4. using HtmlAgilityPack;
  5. using Quartz;
  6. using SCC.Common;
  7. using SCC.Crawler.Tools;
  8. using SCC.Interface;
  9. using SCC.Models;
  10. using SCC.Services;
  11. namespace SCC.Crawler.GP
  12. {
  13. /// <summary>
  14. /// 北京PK10(北京赛车)
  15. /// </summary>
  16. [DisallowConcurrentExecution]
  17. [PersistJobDataAfterExecution]
  18. public class BJPK10Job : IJob
  19. {
  20. /// <summary>
  21. /// 构造函数
  22. /// </summary>
  23. public BJPK10Job()
  24. {
  25. log = new LogHelper();
  26. services = IOC.Resolve<IOpen10Code>();
  27. email = IOC.Resolve<IEmail>();
  28. }
  29. /// <summary>
  30. /// 作业执行入口
  31. /// </summary>
  32. /// <param name="context">作业执行上下文</param>
  33. public void Execute(IJobExecutionContext context)
  34. {
  35. Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
  36. //预设节假日不开奖
  37. if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
  38. LatestItem = context.JobDetail.JobDataMap["LatestItem"] as OpenCode10Model;
  39. try
  40. {
  41. //服务启动时配置初始数据
  42. if (LatestItem == null) LatestItem = services.GetLastItem(currentLottery);
  43. //第一次启动服务或最新开奖记录为昨日滴,则自检昨天开奖数据是否抓取完毕(否则插入邮件数据),并重置当天期号和失败列表
  44. if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") !=
  45. CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
  46. {
  47. CheckingYesterdayTheLotteryData();
  48. LatestItem = GenerateTodayFirstItem();
  49. }
  50. //当最新开奖记录不是今天最后一期,执行当天作业
  51. if (!CheckLatestItemIsTodayLastItem())
  52. {
  53. DoTodayJobByMainUrl();
  54. DoTodayJobByBackUrl();
  55. }
  56. }
  57. catch (Exception ex)
  58. {
  59. log.Error(GetType(), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + currentLottery, ex.Message));
  60. }
  61. //保存最新期号和失败期号列表
  62. context.JobDetail.JobDataMap["LatestItem"] = LatestItem;
  63. }
  64. /// <summary>
  65. /// 自检昨天开奖数据
  66. /// </summary>
  67. private void CheckingYesterdayTheLotteryData()
  68. {
  69. if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd")))
  70. return; //如果昨日设定不开奖则不自检昨日开奖数据
  71. //从数据库中获取昨天数据抓取失败列表
  72. FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, GenerateYesterdayQiHaoList());
  73. if (FailedQiHaoList.Count > 0)
  74. {
  75. DoYesterdayFailedListByMainUrl();
  76. DoYesterdayFailedListByBackUrl();
  77. }
  78. }
  79. /// <summary>
  80. /// 通过主站点抓取开奖数据
  81. /// (百度彩票)
  82. /// </summary>
  83. private void DoTodayJobByMainUrl()
  84. {
  85. if (!string.IsNullOrEmpty(Config.MainUrl))
  86. {
  87. var OpenList = GetDocByMainUrl();
  88. if (OpenList.Count == 0) return; //无抓取数据
  89. var startQiNum = LatestItem.Term + 1;
  90. var newestQiNum = OpenList.Max(w => w.Term);
  91. if (startQiNum > newestQiNum) return; //无最新数据
  92. var total = OpenList.Count;
  93. //处理最新开奖数据
  94. for (var i = startQiNum; i <= newestQiNum; i++)
  95. {
  96. var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
  97. var step = 0;
  98. var nowQiHao = Convert.ToInt32(i);
  99. while (matchItem == null)
  100. if (step <= total)
  101. {
  102. nowQiHao++;
  103. step++;
  104. matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
  105. }
  106. else
  107. {
  108. matchItem = null;
  109. break;
  110. }
  111. if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
  112. {
  113. //处理成功写入日志
  114. log.Info(GetType(), CommonHelper.GetJobMainLogInfo(Config, i.ToString()));
  115. LatestItem = matchItem;
  116. }
  117. }
  118. }
  119. }
  120. /// <summary>
  121. /// 通过主站抓取错误期号列表中每一个期号
  122. /// (百度彩票)
  123. /// </summary>
  124. private void DoYesterdayFailedListByMainUrl()
  125. {
  126. if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
  127. {
  128. var OpenList = GetDocByMainUrl(false);
  129. if (OpenList.Count == 0) return; //无抓取数据
  130. var total = OpenList.Count;
  131. var SuccessList = new List<string>();
  132. foreach (var failedQiHao in FailedQiHaoList)
  133. {
  134. var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
  135. var step = 0;
  136. var nowQiHao = Convert.ToInt32(failedQiHao);
  137. while (matchItem == null)
  138. if (step <= total)
  139. {
  140. nowQiHao++;
  141. step++;
  142. matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
  143. }
  144. else
  145. {
  146. matchItem = null;
  147. break;
  148. }
  149. if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
  150. {
  151. //处理成功写入日志
  152. log.Info(GetType(), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
  153. SuccessList.Add(failedQiHao);
  154. }
  155. }
  156. foreach (var successQiHao in SuccessList) FailedQiHaoList.Remove(successQiHao);
  157. }
  158. }
  159. /// 从主站抓取开奖数据 昨天/今天
  160. /// </summary>
  161. /// <param name="IsToday"></param>
  162. /// <returns></returns>
  163. private List<OpenCode10Model> GetDocByMainUrl(bool IsToday = true)
  164. {
  165. var list = new List<OpenCode10Model>();
  166. try
  167. {
  168. var time = DateTime.Now;
  169. var arg = new OpenCaiApiArg
  170. {
  171. code = EnumHelper.GetLotteryCode(SCCLottery.BeiJingPK10),
  172. date = time.ToString("yyyy-MM-dd")
  173. };
  174. if (!IsToday) arg.date = time.AddDays(-1).ToString("yyyy-MM-dd");
  175. var data = OpenCaiApiServices.GetOpenCaiApiData(arg);
  176. if (data == null) return list;
  177. if (data.data != null)
  178. {
  179. if (data.data.Count == 0) return list;
  180. for (var i = 1; i <= data.data.Count; i++)
  181. {
  182. var tmp = new OpenCode10Model();
  183. var codeAry = data.data[i - 1].GetOpenCodeStr().Split(',');
  184. if (codeAry.Length == 10)
  185. {
  186. tmp.Term = long.Parse(data.data[i - 1].expect);
  187. tmp.OpenTime = DateTime.Parse(data.data[i - 1].opentime);
  188. tmp.OpenCode1 = int.Parse(codeAry[0]);
  189. tmp.OpenCode2 = int.Parse(codeAry[1]);
  190. tmp.OpenCode3 = int.Parse(codeAry[2]);
  191. tmp.OpenCode4 = int.Parse(codeAry[3]);
  192. tmp.OpenCode5 = int.Parse(codeAry[4]);
  193. tmp.OpenCode6 = int.Parse(codeAry[5]);
  194. tmp.OpenCode7 = int.Parse(codeAry[6]);
  195. tmp.OpenCode8 = int.Parse(codeAry[7]);
  196. tmp.OpenCode9 = int.Parse(codeAry[8]);
  197. tmp.OpenCode10 = int.Parse(codeAry[9]);
  198. }
  199. list.Add(tmp);
  200. }
  201. var checkDataHelper = new CheckDataHelper();
  202. var dbdata = services.GetListIn(currentLottery, IsToday);
  203. if (dbdata == null) return list;
  204. var DATA = dbdata.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr());
  205. checkDataHelper.CheckData(DATA, list.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()),
  206. Config.Area, currentLottery);
  207. }
  208. }
  209. catch (Exception ee)
  210. {
  211. log.Error(GetType(),
  212. string.Format("【{0}】通过站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + currentLottery, ee.Message));
  213. }
  214. return list;
  215. }
  216. /// <summary>
  217. /// 通过备用站点抓取开奖数据
  218. /// (北京福彩官网)
  219. /// </summary>
  220. private void DoTodayJobByBackUrl()
  221. {
  222. if (!string.IsNullOrEmpty(Config.BackUrl))
  223. {
  224. var OpenList = GetOpenListFromBackUrl(Config.BackUrl);
  225. if (OpenList.Count == 0) return; //无抓取数据
  226. var startQiNum = LatestItem.Term + 1;
  227. var newestQiNum = OpenList.Max(w => w.Term);
  228. if (startQiNum > newestQiNum) return; //无最新数据
  229. var total = OpenList.Count;
  230. //处理最新开奖数据
  231. for (var i = startQiNum; i <= newestQiNum; i++)
  232. {
  233. var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
  234. var step = 0;
  235. var nowQiHao = Convert.ToInt32(i);
  236. while (matchItem == null)
  237. if (step <= total)
  238. {
  239. nowQiHao++;
  240. step++;
  241. matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
  242. }
  243. else
  244. {
  245. matchItem = null;
  246. break;
  247. }
  248. if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
  249. {
  250. //处理成功写入日志
  251. log.Info(GetType(), CommonHelper.GetJobBackLogInfo(Config, i.ToString()));
  252. LatestItem = matchItem;
  253. }
  254. }
  255. }
  256. }
  257. /// <summary>
  258. /// 通过备用地址抓取错误期号列表中每一个期号
  259. /// (北京福彩官网)
  260. /// </summary>
  261. private void DoYesterdayFailedListByBackUrl()
  262. {
  263. if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
  264. {
  265. var OpenList = GetOpenListFromBackUrl(Config.BackUrl, false);
  266. if (OpenList.Count == 0) return; //无抓取数据
  267. var total = OpenList.Count;
  268. var SuccessList = new List<string>();
  269. foreach (var failedQiHao in FailedQiHaoList)
  270. {
  271. var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
  272. var step = 0;
  273. var nowQiHao = Convert.ToInt32(failedQiHao);
  274. while (matchItem == null)
  275. if (step <= total)
  276. {
  277. nowQiHao++;
  278. step++;
  279. matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
  280. }
  281. else
  282. {
  283. matchItem = null;
  284. break;
  285. }
  286. if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
  287. {
  288. //处理成功写入日志
  289. log.Info(GetType(), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
  290. SuccessList.Add(failedQiHao);
  291. }
  292. }
  293. foreach (var successQiHao in SuccessList) FailedQiHaoList.Remove(successQiHao);
  294. }
  295. }
  296. /// <summary>
  297. /// 抓取备用站点开奖数据
  298. /// 北京福彩官网分页数据太多,只有通过主站点来保证开奖数据完整性
  299. /// </summary>
  300. /// <param name="url">备用站点</param>
  301. /// <returns></returns>
  302. private List<OpenCode10Model> GetOpenListFromBackUrl(string url, bool IsToday = true)
  303. {
  304. var result = new List<OpenCode10Model>();
  305. try
  306. {
  307. var HtmlResource = string.Empty;
  308. HtmlResource = NetHelper.GetUrlResponse(url);
  309. if (HtmlResource == null) return result;
  310. var doc = new HtmlDocument();
  311. doc.LoadHtml(HtmlResource);
  312. var table = doc.DocumentNode.SelectNodes("//table");
  313. if (table == null || table.Count < 2) return result;
  314. var trs = table[1].ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
  315. if (trs.Count < 2) return result;
  316. List<HtmlNode> tds = null;
  317. OpenCode10Model model = null;
  318. string[] openCodeList = null;
  319. var openTime = string.Empty;
  320. var todayDateString = CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd");
  321. for (var i = 1; i < trs.Count; i++) //第一行是表头
  322. {
  323. tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
  324. if (tds == null) return result;
  325. if (tds.Count < 3) continue;
  326. openTime = tds[2].InnerText.Trim();
  327. if (!openTime.StartsWith(todayDateString)) continue;
  328. model = new OpenCode10Model();
  329. model.Term = Convert.ToInt64(tds[0].InnerText.Trim());
  330. openCodeList = tds[1].InnerText.Trim().Split(',');
  331. if (openCodeList.Length < 10) continue;
  332. model.OpenCode1 = Convert.ToInt32(openCodeList[0]);
  333. model.OpenCode2 = Convert.ToInt32(openCodeList[1]);
  334. model.OpenCode3 = Convert.ToInt32(openCodeList[2]);
  335. model.OpenCode4 = Convert.ToInt32(openCodeList[3]);
  336. model.OpenCode5 = Convert.ToInt32(openCodeList[4]);
  337. model.OpenCode6 = Convert.ToInt32(openCodeList[5]);
  338. model.OpenCode7 = Convert.ToInt32(openCodeList[6]);
  339. model.OpenCode8 = Convert.ToInt32(openCodeList[7]);
  340. model.OpenCode9 = Convert.ToInt32(openCodeList[8]);
  341. model.OpenCode10 = Convert.ToInt32(openCodeList[9]);
  342. model.OpenTime = Convert.ToDateTime(openTime);
  343. if (!result.Contains(model))
  344. result.Add(model);
  345. }
  346. var checkDataHelper = new CheckDataHelper();
  347. var dbdata = services.GetListIn(currentLottery, IsToday);
  348. if (dbdata == null) return result;
  349. var datas = dbdata.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr());
  350. checkDataHelper.CheckData(datas, result.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()),
  351. Config.Area, currentLottery);
  352. }
  353. catch (Exception ex)
  354. {
  355. log.Error(GetType(),
  356. string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + currentLottery, ex.Message));
  357. }
  358. return result;
  359. }
  360. /// <summary>
  361. /// 生成今日第一期记录
  362. /// </summary>
  363. /// <returns></returns>
  364. private OpenCode10Model GenerateTodayFirstItem()
  365. {
  366. var item = new OpenCode10Model();
  367. var currentDateTime = CommonHelper.SCCSysDateTime;
  368. var datepart = currentDateTime - new DateTime(2018, 1, 1);
  369. var t = Config.SkipDate.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries);
  370. item.Term = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear; //期号使用昨日最后一期期号,便于计算使用
  371. item.OpenTime = new DateTime(currentDateTime.Year, currentDateTime.Month, currentDateTime.Day,
  372. Config.StartHour, Config.StartMinute, 0);
  373. return item;
  374. }
  375. /// <summary>
  376. /// 核实最新一期开奖记录是否为今日最后一期
  377. /// </summary>
  378. /// <param name="QiHao">期号</param>
  379. /// <returns></returns>
  380. private bool CheckLatestItemIsTodayLastItem()
  381. {
  382. if (LatestItem == null ||
  383. LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
  384. return true;
  385. var firstItem = GenerateTodayFirstItem();
  386. if (LatestItem.Term == firstItem.Term + Config.TimesPerDay)
  387. return true;
  388. return false;
  389. }
  390. /// <summary>
  391. /// 获取今天第一期期号
  392. /// </summary>
  393. /// <returns></returns>
  394. private string GenerateTodayFirstQiHao()
  395. {
  396. var datepart = CommonHelper.SCCSysDateTime - new DateTime(2018, 1, 1);
  397. var t = Config.SkipDate.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries);
  398. var terms = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear + 1;
  399. return terms.ToString();
  400. }
  401. /// <summary>
  402. /// 生成昨天的期号列表
  403. /// </summary>
  404. /// <returns></returns>
  405. private List<string> GenerateYesterdayQiHaoList()
  406. {
  407. var result = new List<string>();
  408. var todayFirstQiHao = Convert.ToInt64(GenerateTodayFirstQiHao());
  409. for (var i = Config.TimesPerDay; i > 0; i--) result.Add((todayFirstQiHao - i).ToString());
  410. return result;
  411. }
  412. /// <summary>
  413. /// 生成PK10的开奖时间
  414. /// </summary>
  415. /// <param name="QiHao"></param>
  416. /// <returns></returns>
  417. private DateTime GenerateYesterdayPK10OpenTime(string QiHao)
  418. {
  419. var openDay = CommonHelper.SCCSysDateTime.AddDays(-1);
  420. var StartTime = new DateTime(openDay.Year, openDay.Month, openDay.Day, Config.StartHour, Config.StartMinute,
  421. 0);
  422. var t = (Convert.ToInt64(QiHao) - LastTermLastYear) % Config.TimesPerDay;
  423. if (t == 0)
  424. return StartTime.AddMinutes((Config.TimesPerDay - 1) * Config.Interval);
  425. return StartTime.AddMinutes((t - 1) * Config.Interval);
  426. }
  427. #region Attribute
  428. /// <summary>
  429. /// 配置信息
  430. /// </summary>
  431. private SCCConfig Config;
  432. /// <summary>
  433. /// 当天抓取的最新一期开奖记录
  434. /// </summary>
  435. private OpenCode10Model LatestItem;
  436. /// <summary>
  437. /// 当天抓取失败列表
  438. /// </summary>
  439. private List<string> FailedQiHaoList;
  440. /// <summary>
  441. /// 日志对象
  442. /// </summary>
  443. private readonly LogHelper log;
  444. /// <summary>
  445. /// 数据服务
  446. /// </summary>
  447. private readonly IOpen10Code services;
  448. /// <summary>
  449. /// 当前彩种
  450. /// </summary>
  451. private SCCLottery currentLottery => SCCLottery.BeiJingPK10;
  452. /// <summary>
  453. /// 邮件接口
  454. /// </summary>
  455. private IEmail email;
  456. /// <summary>
  457. /// 2016年最后一期期号
  458. /// </summary>
  459. private readonly long LastTermLastYear = 659223;
  460. #endregion
  461. }
  462. }