123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507 |
- using System;
- using System.Collections.Generic;
- using System.Linq;
- using HtmlAgilityPack;
- using Quartz;
- using SCC.Common;
- using SCC.Crawler.Tools;
- using SCC.Interface;
- using SCC.Models;
- using SCC.Services;
- namespace SCC.Crawler.GP
- {
-
-
-
- [DisallowConcurrentExecution]
- [PersistJobDataAfterExecution]
- public class BJPK10Job : IJob
- {
-
-
-
- public BJPK10Job()
- {
- log = new LogHelper();
- services = IOC.Resolve<IOpen10Code>();
- email = IOC.Resolve<IEmail>();
- }
-
-
-
-
- public void Execute(IJobExecutionContext context)
- {
- Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
-
- if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))) return;
- LatestItem = context.JobDetail.JobDataMap["LatestItem"] as OpenCode10Model;
- try
- {
-
- if (LatestItem == null) LatestItem = services.GetLastItem(currentLottery);
-
- if (LatestItem == null || LatestItem.OpenTime.ToString("yyyyMMdd") !=
- CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
- {
- CheckingYesterdayTheLotteryData();
- LatestItem = GenerateTodayFirstItem();
- }
-
- if (!CheckLatestItemIsTodayLastItem())
- {
- DoTodayJobByMainUrl();
- DoTodayJobByBackUrl();
- }
- }
- catch (Exception ex)
- {
- log.Error(GetType(), string.Format("【{0}】抓取时发生错误,错误信息【{1}】", Config.Area + currentLottery, ex.Message));
- }
-
- context.JobDetail.JobDataMap["LatestItem"] = LatestItem;
- }
-
-
-
- private void CheckingYesterdayTheLotteryData()
- {
- if (Config.SkipDate.Contains(CommonHelper.SCCSysDateTime.AddDays(-1).ToString("yyyyMMdd")))
- return;
-
- FailedQiHaoList = services.GetYesterdayFailQQQList(currentLottery, GenerateYesterdayQiHaoList());
- if (FailedQiHaoList.Count > 0)
- {
- DoYesterdayFailedListByMainUrl();
- DoYesterdayFailedListByBackUrl();
- }
- }
-
-
-
-
- private void DoTodayJobByMainUrl()
- {
- if (!string.IsNullOrEmpty(Config.MainUrl))
- {
- var OpenList = GetDocByMainUrl();
- if (OpenList.Count == 0) return;
- var startQiNum = LatestItem.Term + 1;
- var newestQiNum = OpenList.Max(w => w.Term);
- if (startQiNum > newestQiNum) return;
- var total = OpenList.Count;
-
- for (var i = startQiNum; i <= newestQiNum; i++)
- {
- var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
- var step = 0;
- var nowQiHao = Convert.ToInt32(i);
- while (matchItem == null)
- if (step <= total)
- {
- nowQiHao++;
- step++;
- matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
- }
- else
- {
- matchItem = null;
- break;
- }
- if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
- {
-
- log.Info(GetType(), CommonHelper.GetJobMainLogInfo(Config, i.ToString()));
- LatestItem = matchItem;
- }
- }
- }
- }
-
-
-
-
- private void DoYesterdayFailedListByMainUrl()
- {
- if (!string.IsNullOrEmpty(Config.MainUrl) && FailedQiHaoList.Count > 0)
- {
- var OpenList = GetDocByMainUrl(false);
- if (OpenList.Count == 0) return;
- var total = OpenList.Count;
- var SuccessList = new List<string>();
- foreach (var failedQiHao in FailedQiHaoList)
- {
- var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
- var step = 0;
- var nowQiHao = Convert.ToInt32(failedQiHao);
- while (matchItem == null)
- if (step <= total)
- {
- nowQiHao++;
- step++;
- matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
- }
- else
- {
- matchItem = null;
- break;
- }
- if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
- {
-
- log.Info(GetType(), CommonHelper.GetJobMainLogInfo(Config, failedQiHao));
- SuccessList.Add(failedQiHao);
- }
- }
- foreach (var successQiHao in SuccessList) FailedQiHaoList.Remove(successQiHao);
- }
- }
-
-
-
-
- private List<OpenCode10Model> GetDocByMainUrl(bool IsToday = true)
- {
- var list = new List<OpenCode10Model>();
- try
- {
- var time = DateTime.Now;
- var arg = new OpenCaiApiArg
- {
- code = EnumHelper.GetLotteryCode(SCCLottery.BeiJingPK10),
- date = time.ToString("yyyy-MM-dd")
- };
- if (!IsToday) arg.date = time.AddDays(-1).ToString("yyyy-MM-dd");
- var data = OpenCaiApiServices.GetOpenCaiApiData(arg);
- if (data == null) return list;
- if (data.data != null)
- {
- if (data.data.Count == 0) return list;
- for (var i = 1; i <= data.data.Count; i++)
- {
- var tmp = new OpenCode10Model();
- var codeAry = data.data[i - 1].GetOpenCodeStr().Split(',');
- if (codeAry.Length == 10)
- {
- tmp.Term = long.Parse(data.data[i - 1].expect);
- tmp.OpenTime = DateTime.Parse(data.data[i - 1].opentime);
- tmp.OpenCode1 = int.Parse(codeAry[0]);
- tmp.OpenCode2 = int.Parse(codeAry[1]);
- tmp.OpenCode3 = int.Parse(codeAry[2]);
- tmp.OpenCode4 = int.Parse(codeAry[3]);
- tmp.OpenCode5 = int.Parse(codeAry[4]);
- tmp.OpenCode6 = int.Parse(codeAry[5]);
- tmp.OpenCode7 = int.Parse(codeAry[6]);
- tmp.OpenCode8 = int.Parse(codeAry[7]);
- tmp.OpenCode9 = int.Parse(codeAry[8]);
- tmp.OpenCode10 = int.Parse(codeAry[9]);
- }
- list.Add(tmp);
- }
- var checkDataHelper = new CheckDataHelper();
- var dbdata = services.GetListIn(currentLottery, IsToday);
- if (dbdata == null) return list;
- var DATA = dbdata.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr());
- checkDataHelper.CheckData(DATA, list.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()),
- Config.Area, currentLottery);
- }
- }
- catch (Exception ee)
- {
- log.Error(GetType(),
- string.Format("【{0}】通过站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + currentLottery, ee.Message));
- }
- return list;
- }
-
-
-
-
- private void DoTodayJobByBackUrl()
- {
- if (!string.IsNullOrEmpty(Config.BackUrl))
- {
- var OpenList = GetOpenListFromBackUrl(Config.BackUrl);
- if (OpenList.Count == 0) return;
- var startQiNum = LatestItem.Term + 1;
- var newestQiNum = OpenList.Max(w => w.Term);
- if (startQiNum > newestQiNum) return;
- var total = OpenList.Count;
-
- for (var i = startQiNum; i <= newestQiNum; i++)
- {
- var matchItem = OpenList.Where(R => R.Term == i).FirstOrDefault();
- var step = 0;
- var nowQiHao = Convert.ToInt32(i);
- while (matchItem == null)
- if (step <= total)
- {
- nowQiHao++;
- step++;
- matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
- }
- else
- {
- matchItem = null;
- break;
- }
- if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
- {
-
- log.Info(GetType(), CommonHelper.GetJobBackLogInfo(Config, i.ToString()));
- LatestItem = matchItem;
- }
- }
- }
- }
-
-
-
-
- private void DoYesterdayFailedListByBackUrl()
- {
- if (!string.IsNullOrEmpty(Config.BackUrl) && FailedQiHaoList.Count > 0)
- {
- var OpenList = GetOpenListFromBackUrl(Config.BackUrl, false);
- if (OpenList.Count == 0) return;
- var total = OpenList.Count;
- var SuccessList = new List<string>();
- foreach (var failedQiHao in FailedQiHaoList)
- {
- var matchItem = OpenList.Where(R => R.Term.ToString() == failedQiHao).FirstOrDefault();
- var step = 0;
- var nowQiHao = Convert.ToInt32(failedQiHao);
- while (matchItem == null)
- if (step <= total)
- {
- nowQiHao++;
- step++;
- matchItem = OpenList.Where(R => R.Term.ToString() == nowQiHao.ToString()).FirstOrDefault();
- }
- else
- {
- matchItem = null;
- break;
- }
- if (matchItem != null && services.AddOpen10Code(currentLottery, matchItem))
- {
-
- log.Info(GetType(), CommonHelper.GetJobBackLogInfo(Config, failedQiHao));
- SuccessList.Add(failedQiHao);
- }
- }
- foreach (var successQiHao in SuccessList) FailedQiHaoList.Remove(successQiHao);
- }
- }
-
-
-
-
-
-
- private List<OpenCode10Model> GetOpenListFromBackUrl(string url, bool IsToday = true)
- {
- var result = new List<OpenCode10Model>();
- try
- {
- var HtmlResource = string.Empty;
- HtmlResource = NetHelper.GetUrlResponse(url);
- if (HtmlResource == null) return result;
- var doc = new HtmlDocument();
- doc.LoadHtml(HtmlResource);
- var table = doc.DocumentNode.SelectNodes("//table");
- if (table == null || table.Count < 2) return result;
- var trs = table[1].ChildNodes.Where(R => R.Name.ToLower() == "tr").ToList();
- if (trs.Count < 2) return result;
- List<HtmlNode> tds = null;
- OpenCode10Model model = null;
- string[] openCodeList = null;
- var openTime = string.Empty;
- var todayDateString = CommonHelper.SCCSysDateTime.ToString("yyyy-MM-dd");
- for (var i = 1; i < trs.Count; i++)
- {
- tds = trs[i].ChildNodes.Where(R => R.Name.ToLower() == "td").ToList();
- if (tds == null) return result;
- if (tds.Count < 3) continue;
- openTime = tds[2].InnerText.Trim();
- if (!openTime.StartsWith(todayDateString)) continue;
- model = new OpenCode10Model();
- model.Term = Convert.ToInt64(tds[0].InnerText.Trim());
- openCodeList = tds[1].InnerText.Trim().Split(',');
- if (openCodeList.Length < 10) continue;
- model.OpenCode1 = Convert.ToInt32(openCodeList[0]);
- model.OpenCode2 = Convert.ToInt32(openCodeList[1]);
- model.OpenCode3 = Convert.ToInt32(openCodeList[2]);
- model.OpenCode4 = Convert.ToInt32(openCodeList[3]);
- model.OpenCode5 = Convert.ToInt32(openCodeList[4]);
- model.OpenCode6 = Convert.ToInt32(openCodeList[5]);
- model.OpenCode7 = Convert.ToInt32(openCodeList[6]);
- model.OpenCode8 = Convert.ToInt32(openCodeList[7]);
- model.OpenCode9 = Convert.ToInt32(openCodeList[8]);
- model.OpenCode10 = Convert.ToInt32(openCodeList[9]);
- model.OpenTime = Convert.ToDateTime(openTime);
- if (!result.Contains(model))
- result.Add(model);
- }
- var checkDataHelper = new CheckDataHelper();
- var dbdata = services.GetListIn(currentLottery, IsToday);
- if (dbdata == null) return result;
- var datas = dbdata.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr());
- checkDataHelper.CheckData(datas, result.ToDictionary(w => w.Term.ToString(), w => w.GetCodeStr()),
- Config.Area, currentLottery);
- }
- catch (Exception ex)
- {
- log.Error(GetType(),
- string.Format("【{0}】通过备用站点抓取开奖列表时发生错误,错误信息【{1}】", Config.Area + currentLottery, ex.Message));
- }
- return result;
- }
-
-
-
-
- private OpenCode10Model GenerateTodayFirstItem()
- {
- var item = new OpenCode10Model();
- var currentDateTime = CommonHelper.SCCSysDateTime;
- var datepart = currentDateTime - new DateTime(2018, 1, 1);
- var t = Config.SkipDate.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries);
- item.Term = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear;
- item.OpenTime = new DateTime(currentDateTime.Year, currentDateTime.Month, currentDateTime.Day,
- Config.StartHour, Config.StartMinute, 0);
- return item;
- }
-
-
-
-
-
- private bool CheckLatestItemIsTodayLastItem()
- {
- if (LatestItem == null ||
- LatestItem.OpenTime.ToString("yyyyMMdd") != CommonHelper.SCCSysDateTime.ToString("yyyyMMdd"))
- return true;
- var firstItem = GenerateTodayFirstItem();
- if (LatestItem.Term == firstItem.Term + Config.TimesPerDay)
- return true;
- return false;
- }
-
-
-
-
- private string GenerateTodayFirstQiHao()
- {
- var datepart = CommonHelper.SCCSysDateTime - new DateTime(2018, 1, 1);
- var t = Config.SkipDate.Split(new[] {','}, StringSplitOptions.RemoveEmptyEntries);
- var terms = (datepart.Days - t.Length) * Config.TimesPerDay + LastTermLastYear + 1;
- return terms.ToString();
- }
-
-
-
-
- private List<string> GenerateYesterdayQiHaoList()
- {
- var result = new List<string>();
- var todayFirstQiHao = Convert.ToInt64(GenerateTodayFirstQiHao());
- for (var i = Config.TimesPerDay; i > 0; i--) result.Add((todayFirstQiHao - i).ToString());
- return result;
- }
-
-
-
-
-
- private DateTime GenerateYesterdayPK10OpenTime(string QiHao)
- {
- var openDay = CommonHelper.SCCSysDateTime.AddDays(-1);
- var StartTime = new DateTime(openDay.Year, openDay.Month, openDay.Day, Config.StartHour, Config.StartMinute,
- 0);
- var t = (Convert.ToInt64(QiHao) - LastTermLastYear) % Config.TimesPerDay;
- if (t == 0)
- return StartTime.AddMinutes((Config.TimesPerDay - 1) * Config.Interval);
- return StartTime.AddMinutes((t - 1) * Config.Interval);
- }
- #region Attribute
-
-
-
- private SCCConfig Config;
-
-
-
- private OpenCode10Model LatestItem;
-
-
-
- private List<string> FailedQiHaoList;
-
-
-
- private readonly LogHelper log;
-
-
-
- private readonly IOpen10Code services;
-
-
-
- private SCCLottery currentLottery => SCCLottery.BeiJingPK10;
-
-
-
- private IEmail email;
-
-
-
- private readonly long LastTermLastYear = 659223;
- #endregion
- }
- }
|