CommonHelper.cs 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863
  1. using FCS.Models;
  2. using FCS.Models.DTO;
  3. using HtmlAgilityPack;
  4. using Newtonsoft.Json;
  5. using Quartz;
  6. using System;
  7. using System.Collections.Generic;
  8. using System.Configuration;
  9. using System.Diagnostics;
  10. using System.IO;
  11. using System.Linq;
  12. using System.Net;
  13. using System.Reflection;
  14. using System.Runtime.Remoting.Messaging;
  15. using System.Text;
  16. using System.Text.RegularExpressions;
  17. using System.Threading;
  18. using System.Threading.Tasks;
  19. using System.Xml;
  20. namespace FCS.Common
  21. {
  22. /// <summary>
  23. /// 公用帮助类
  24. /// </summary>
  25. public static class CommonHelper
  26. {
  27. /// <summary>
  28. /// 将XML内容转换成目标对象实体集合
  29. /// </summary>
  30. /// <typeparam name="T">目标对象实体</typeparam>
  31. /// <param name="FileName">完整文件名(根目录下只需文件名称)</param>
  32. /// <param name="WrapperNodeName"></param>
  33. /// <returns></returns>
  34. public static List<T> ConvertXMLToObject<T>(string FileName, string WrapperNodeName)
  35. {
  36. XmlDocument doc = new XmlDocument();
  37. doc.Load(FileName);
  38. List<T> result = new List<T>();
  39. var TType = typeof(T);
  40. XmlNodeList nodeList = doc.ChildNodes;
  41. if (!string.IsNullOrEmpty(WrapperNodeName))
  42. {
  43. foreach (XmlNode node in doc.ChildNodes)
  44. {
  45. if (node.Name == WrapperNodeName)
  46. {
  47. nodeList = node.ChildNodes;
  48. break;
  49. }
  50. }
  51. }
  52. object oneT = null;
  53. foreach (XmlNode node in nodeList)
  54. {
  55. if (node.NodeType == XmlNodeType.Comment || node.NodeType == XmlNodeType.XmlDeclaration) continue;
  56. oneT = TType.Assembly.CreateInstance(TType.FullName);
  57. foreach (XmlNode item in node.ChildNodes)
  58. {
  59. if (item.NodeType == XmlNodeType.Comment) continue;
  60. var property = TType.GetProperty(item.Name);
  61. if (property != null)
  62. property.SetValue(oneT, Convert.ChangeType(item.InnerText, property.PropertyType), null);
  63. }
  64. result.Add((T)oneT);
  65. }
  66. return result;
  67. }
  68. /// <summary>
  69. /// 从作业数据地图中获取配置信息
  70. /// </summary>
  71. /// <param name="datamap">作业数据地图</param>
  72. /// <returns></returns>
  73. public static FCSConfig GetConfigFromDataMap(JobDataMap datamap)
  74. {
  75. FCSConfig config = new FCSConfig();
  76. var properties = typeof(FCSConfig).GetProperties();
  77. foreach (PropertyInfo info in properties)
  78. {
  79. if (info.PropertyType == typeof(string))
  80. info.SetValue(config, datamap.GetString(info.Name), null);
  81. else if (info.PropertyType == typeof(Int32))
  82. info.SetValue(config, datamap.GetInt(info.Name), null);
  83. }
  84. return config;
  85. }
  86. #region 日志信息
  87. public static string GetJobMainLogInfo(string QiHao)
  88. {
  89. return string.Format("通过主站地址抓取{0}期开奖数据成功", QiHao);
  90. }
  91. public static string GetJobLogError(string QiHao)
  92. {
  93. return string.Format("【{0}】抓取期开奖数据失败", QiHao);
  94. }
  95. #endregion 日志信息
  96. /// <summary>
  97. /// 将值转换为T类型数据
  98. /// </summary>
  99. /// <typeparam name="T">目标类型</typeparam>
  100. /// <param name="value">数据值</param>
  101. /// <returns></returns>
  102. public static T ChangeType<T>(object value)
  103. {
  104. return ChangeType<T>(value, default(T));
  105. }
  106. /// <summary>
  107. /// 将值转换为T类型数据,失败则返回T类型默认值
  108. /// </summary>
  109. /// <typeparam name="T">目标类型</typeparam>
  110. /// <param name="value">数据值</param>
  111. /// <param name="defaultValue">T类型默认值</param>
  112. /// <returns></returns>
  113. public static T ChangeType<T>(object value, T defaultValue)
  114. {
  115. if (value != null)
  116. {
  117. Type nullableType = typeof(T);
  118. if (!nullableType.IsInterface && (!nullableType.IsClass || (nullableType == typeof(string))))
  119. {
  120. if (nullableType.IsGenericType && (nullableType.GetGenericTypeDefinition() == typeof(Nullable<>)))
  121. {
  122. return (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(nullableType));
  123. }
  124. if (nullableType.IsEnum)
  125. {
  126. return (T)Enum.Parse(nullableType, value.ToString());
  127. }
  128. return (T)Convert.ChangeType(value, nullableType);
  129. }
  130. if (value is T)
  131. {
  132. return (T)value;
  133. }
  134. }
  135. return defaultValue;
  136. }
  137. /// <summary>
  138. /// 将值转换为type类型的值
  139. /// </summary>
  140. /// <param name="value"></param>
  141. /// <param name="type">目标类型</param>
  142. /// <returns></returns>
  143. public static object ChangeType(object value, Type type)
  144. {
  145. if (value != null)
  146. {
  147. var nullableType = Nullable.GetUnderlyingType(type);
  148. if (nullableType != null)//可空
  149. {
  150. return Convert.ChangeType(value, nullableType);
  151. }
  152. if (Convert.IsDBNull(value))//特殊处理,由于数据库类型与项目中的类型定义不匹配
  153. return type.IsValueType ? Activator.CreateInstance(type) : null;
  154. return Convert.ChangeType(value, type);
  155. }
  156. return null;
  157. }
  158. #region 获取全局唯一GUID
  159. /// <summary>
  160. /// 获取全局唯一GUID
  161. /// </summary>
  162. /// <param name="needReplace">是否需要替换-</param>
  163. /// <param name="format">格式化</param>
  164. /// <example>N:38bddf48f43c48588e0d78761eaa1ce6</example>>
  165. /// <example>P:(778406c2-efff-4262-ab03-70a77d09c2b5)</example>>
  166. /// <example>B:{09f140d5-af72-44ba-a763-c861304b46f8}</example>>
  167. /// <example>D:57d99d89-caab-482a-a0e9-a0a803eed3ba</example>>
  168. /// <returns></returns>
  169. public static string GetGuid(bool needReplace = true, string format = "N")
  170. {
  171. Guid res = NewSequentialGuid();//Guid.NewGuid();
  172. return needReplace ? res.ToString(format) : res.ToString();
  173. }
  174. [System.Runtime.InteropServices.DllImport("rpcrt4.dll", SetLastError = true)]
  175. static extern int UuidCreateSequential(byte[] buffer);
  176. /// <summary>
  177. /// 创建有序GUID
  178. /// </summary>
  179. /// <returns></returns>
  180. private static Guid NewSequentialGuid()
  181. {
  182. byte[] raw = new byte[16];
  183. if (UuidCreateSequential(raw) != 0)
  184. throw new System.ComponentModel.Win32Exception(System.Runtime.InteropServices.Marshal.GetLastWin32Error());
  185. byte[] fix = new byte[16];
  186. // reverse 0..3
  187. fix[0x0] = raw[0x3];
  188. fix[0x1] = raw[0x2];
  189. fix[0x2] = raw[0x1];
  190. fix[0x3] = raw[0x0];
  191. // reverse 4 & 5
  192. fix[0x4] = raw[0x5];
  193. fix[0x5] = raw[0x4];
  194. // reverse 6 & 7
  195. fix[0x6] = raw[0x7];
  196. fix[0x7] = raw[0x6];
  197. // all other are unchanged
  198. fix[0x8] = raw[0x8];
  199. fix[0x9] = raw[0x9];
  200. fix[0xA] = raw[0xA];
  201. fix[0xB] = raw[0xB];
  202. fix[0xC] = raw[0xC];
  203. fix[0xD] = raw[0xD];
  204. fix[0xE] = raw[0xE];
  205. fix[0xF] = raw[0xF];
  206. return new Guid(fix);
  207. }
  208. #endregion 获取全局唯一GUID
  209. #region HtmlAgilityPack
  210. public static int IpCount;
  211. public static List<string> IpList;
  212. public static string GetIp(string path = "", bool isGetIp = true)
  213. {
  214. if (isGetIp && IpList.Count > 0)
  215. {
  216. var ran = new Random().Next(0, IpList.Count);
  217. return IpList[ran];
  218. }
  219. if (path.IsEmpty())
  220. path = AppDomain.CurrentDomain.BaseDirectory + "/XmlConfig/IP.txt";
  221. StreamReader sr;
  222. try
  223. {
  224. sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
  225. }
  226. catch (Exception)
  227. {
  228. Thread.Sleep(1000);
  229. sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
  230. }
  231. string content = sr.ReadToEnd().ToString();
  232. sr.Close();
  233. var list = content.JsonToList<string>();
  234. if (isGetIp)
  235. IpList = list;
  236. var ram = new Random().Next(0, list.Count);
  237. IpCount = list.Count;
  238. return list[ram];
  239. }
  240. static object locker = new object();
  241. static bool isGetIp = false;
  242. public delegate string GetIPDataBYOne(List<string> _urlList, string _title = "", bool isFormData = false);
  243. public delegate string GetIPDataBYOne_FormData(List<string> _urlList, Dictionary<string, string> formData, string _title = "");
  244. /// <summary>
  245. /// 获取HTML
  246. /// </summary>
  247. /// <param name="model">参数实体</param>
  248. /// <returns></returns>
  249. public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model)
  250. {
  251. var doc = new HtmlDocument();
  252. doc.LoadHtml(GetHtmlByIP(model));
  253. return doc;
  254. }
  255. /// <summary>
  256. /// 获取HTML
  257. /// </summary>
  258. /// <param name="model">参数实体</param>
  259. /// <returns></returns>
  260. public static string GetHtmlString(HtmlParameterDTO model)
  261. {
  262. return GetHtmlByIP(model);
  263. }
  264. /// <summary>
  265. /// 通过Ip获取页面的HTML
  266. /// </summary>
  267. /// <param name="model"></param>
  268. /// <returns></returns>
  269. private static string GetHtmlByIP(HtmlParameterDTO model)
  270. {
  271. var NotIpList = new List<string>();
  272. Stopwatch sw = new Stopwatch();
  273. sw.Start();
  274. var ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  275. var httpItem = new HttpItem();
  276. lock (locker)
  277. {
  278. httpItem=Mapper<HttpItem>(model);
  279. httpItem.WebProxy = new WebProxy(ip);
  280. }
  281. var html = new HttpHelper().GetHtml(httpItem);
  282. //对文本的检查
  283. while ((model.IsCheckEmpty && html.Html.IsEmpty())
  284. || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
  285. || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
  286. || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
  287. || html.Html.Contains("502 Bad Gateway")
  288. || html.Html.Contains("400 Bad Request")
  289. || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
  290. || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
  291. || html.Html.Contains("缓存访问被拒绝")
  292. || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title)))
  293. {
  294. if (html.Html.ToLower().Contains("exception report"))
  295. {
  296. return ConfigurationManager.AppSettings["Termination"].ToString();
  297. }
  298. NotIpList.Add(ip);
  299. if (NotIpList.Distinct().ToList().Count == IpCount || NotIpList.Distinct().ToList().Count > model.NotIpNumber)
  300. {
  301. //EmailHelper.Send("1625453870@qq.com", "IP用完,未获取值的URL", "URL:" + model.Url + "||参数:" + model.FormData.TryToJson());
  302. //return ConfigurationManager.AppSettings["Termination"].ToString();
  303. LogBD(model.Url, "LogUrl", "UrlLog");
  304. return ConfigurationManager.AppSettings["Termination"].ToString();
  305. //IAsyncResult asyncResult;
  306. //lock (locker)
  307. //{
  308. // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
  309. // asyncResult = task.BeginInvoke(new List<string> { model.Url }, model.Title, false, null, null);
  310. // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
  311. // {
  312. // }
  313. // ip = task.EndInvoke(asyncResult);
  314. // if (ip.IsEmpty())
  315. // return ConfigurationManager.AppSettings["Termination"].ToString();
  316. // else
  317. // return ip;
  318. //}
  319. }
  320. else
  321. {
  322. ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  323. while (NotIpList.Contains(ip) && model.IP.IsEmpty())
  324. ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  325. }
  326. httpItem.WebProxy = new WebProxy(ip);
  327. html = new HttpHelper().GetHtml(httpItem);
  328. }
  329. sw.Stop();
  330. Trace.WriteLine("url:" + model.Url + "||IP:" + ip + "||时间:" + sw.ElapsedMilliseconds + "毫秒");
  331. return html.Html;
  332. }
  333. /// <summary>
  334. /// 得到HtmlDocument
  335. /// </summary>
  336. /// <param name="url"></param>
  337. /// <param name="method"></param>
  338. /// <returns></returns>
  339. public static HtmlDocument GetHtml(string url, string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get", int timeout = 90 * 1000, int notIpNUmber = 100)
  340. {
  341. return GetHtmlHtmlDocument(new HtmlParameterDTO
  342. {
  343. Url = url,
  344. Title = title,
  345. IP = webProxy,
  346. Method = method,
  347. Timeout = timeout,
  348. NotIpNumber = notIpNUmber
  349. });
  350. }
  351. /// <summary>
  352. /// 得到HtmlDocument
  353. /// From表单提交
  354. /// </summary>
  355. /// <param name="url"></param>
  356. /// <param name="method"></param>
  357. /// <returns></returns>
  358. public static HtmlDocument GetHtml(string url, Dictionary<string, string> formData, string title = "", string webProxy = "", int timeout = 90 * 1000, int notIpNUmber = 100)
  359. {
  360. // ContentType = "application/x-www-form-urlencoded",
  361. return GetHtmlHtmlDocument(new HtmlParameterDTO
  362. {
  363. Url = url,
  364. Title = title,
  365. IP = webProxy,
  366. Method = "POST",
  367. Timeout = timeout,
  368. NotIpNumber = notIpNUmber,
  369. ContentType = "application/x-www-form-urlencoded",
  370. FormData = formData
  371. });
  372. }
  373. #endregion
  374. #region lg
  375. /// <summary>
  376. /// 得到HtmlDocument
  377. /// </summary>
  378. /// <param name="url"></param>
  379. /// <param name="method"></param>
  380. /// <returns></returns>
  381. public static string GetHtmlString(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
  382. {
  383. var NotIpList = new List<string>();
  384. var ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  385. var html = new HttpHelper().GetHtml(new HttpItem
  386. {
  387. Url = url,
  388. Method = method,
  389. WebProxy = new WebProxy(ip),
  390. Timeout = timeout
  391. });
  392. int number = 0;
  393. if (!isWebSoxket)
  394. {
  395. while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  396. || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  397. || (!title.IsEmpty() && html.Html.IndexOf(title) == -1)
  398. || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
  399. || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
  400. || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
  401. || html.Html.Contains("502 Bad Gateway")
  402. || html.Html.Contains("400 Bad Request")
  403. || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
  404. || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
  405. || html.Html.Contains("缓存访问被拒绝")
  406. || (title.IsEmpty() && !html.Html.Contains(title)))
  407. {
  408. number++;
  409. if (number > 40)
  410. return "";
  411. if (html.Html.ToLower().Contains("exception report"))
  412. {
  413. ConfigurationManager.AppSettings["Termination"].ToString();
  414. break;
  415. }
  416. NotIpList.Add(ip);
  417. if (NotIpList.Distinct().ToList().Count == IpCount)
  418. {
  419. //IAsyncResult asyncResult;
  420. //lock (locker)
  421. //{
  422. // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
  423. // asyncResult = task.BeginInvoke(new List<string> { url }, title, false, null, null);
  424. // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
  425. // {
  426. // }
  427. // ip = task.EndInvoke(asyncResult);
  428. // if (ip.IsEmpty())
  429. // {
  430. return ConfigurationManager.AppSettings["Termination"].ToString();
  431. // break;
  432. // }
  433. //}
  434. //}
  435. }
  436. else
  437. ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  438. while (NotIpList.Contains(ip))
  439. ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  440. html = new HttpHelper().GetHtml(new HttpItem
  441. {
  442. Url = url,
  443. Method = method,
  444. WebProxy = new WebProxy(ip)
  445. });
  446. }
  447. }
  448. return html.Html;
  449. }
  450. /// <summary>
  451. /// 得到HtmlDocument
  452. /// </summary>
  453. /// <param name="url"></param>
  454. /// <param name="method"></param>
  455. /// <returns></returns>
  456. public static HttpResult GetPostHtmlString(string url, HttpItem model, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
  457. {
  458. model.Timeout = timeout;
  459. var html = new HttpHelper().GetHtml(model);
  460. if (!isWebSoxket)
  461. {
  462. var number = 0;
  463. while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  464. || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  465. || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
  466. {
  467. number++;
  468. if (number > 20)
  469. return html;
  470. model.WebProxy = new WebProxy(GetIp());
  471. html = new HttpHelper().GetHtml(model);
  472. }
  473. }
  474. return html;
  475. }
  476. public static string GetHtmlString_ceshi(string url = "http://fenxi.zgzcw.com/2321966/bjop", string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get")
  477. {
  478. var html3 = new HttpHelper().GetHtml(new HttpItem
  479. {
  480. Url = url,
  481. Method = method,
  482. WebProxy = new WebProxy("113.124.93.1:601")
  483. });
  484. html3 = new HttpHelper().GetHtml(new HttpItem
  485. {
  486. Url = url,
  487. Method = method,
  488. WebProxy = new WebProxy("117.91.249.95:601")
  489. });
  490. F_Grouping g = new F_Grouping();
  491. List<string> list = new List<string>();
  492. list.Add("27.26.162.129");
  493. list.Add("117.91.249.95");
  494. list.Add("222.189.190.117");
  495. list.Add("222.189.191.254");
  496. list.Add("111.72.57.234");
  497. list.Add("113.124.93.1");
  498. list.Add("60.189.167.102");
  499. list.Add("180.118.141.126");
  500. list.Add("221.230.123.45");
  501. list.Add("221.230.123.101");
  502. list.Add("221.230.124.127");
  503. list.Add("125.125.45.149");
  504. list.Add("182.34.32.90");
  505. list.Add("113.121.46.19");
  506. list.Add("113.121.45.103");
  507. list.Add("113.121.23.219");
  508. list.Add("111.72.56.135");
  509. list.Add("111.72.63.12");
  510. list.Add("111.72.62.174");
  511. list.Add("111.72.58.28");
  512. list.Add("111.72.62.226");
  513. list.Add("106.226.227.245");
  514. list.Add("111.79.173.163");
  515. list.Add("106.7.78.39");
  516. list.Add("182.84.86.158");
  517. list.Add("182.100.238.11");
  518. list.Add("111.72.107.203");
  519. List<int> ss = new List<int>();
  520. for (int i = 0; i < 65535; i++)
  521. {
  522. ss.Add(i + 1);
  523. }
  524. int max1 = list.Count;
  525. int num1 = 0;
  526. list.ForEach(async p =>
  527. {
  528. await Task.Run(() =>
  529. {
  530. int max = 65535;
  531. int num = 0;
  532. //比赛
  533. ss.ForEach(async p1 =>
  534. {
  535. await Task.Run(() =>
  536. {
  537. var html = new HttpHelper().GetHtml(new HttpItem
  538. {
  539. Url = url,
  540. Method = method,
  541. WebProxy = new WebProxy(p + ":" + p1)
  542. });
  543. });
  544. lock (g)
  545. {
  546. num++;
  547. Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
  548. }
  549. });
  550. lock (g)
  551. {
  552. while (num < max)
  553. {
  554. Monitor.Wait(g);//等待
  555. }
  556. }
  557. });
  558. lock (g)
  559. {
  560. num1++;
  561. Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
  562. }
  563. });
  564. lock (g)
  565. {
  566. while (num1 < max1)
  567. {
  568. Monitor.Wait(g);//等待
  569. }
  570. }
  571. //if (!isWebSoxket)
  572. //{
  573. // var number = 0;
  574. // while (html.Html.IsEmpty() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  575. // || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  576. // || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
  577. // {
  578. // number++;
  579. // if (number > 100)
  580. // return null;
  581. // //if (number > IpCount)
  582. // html = new HttpHelper().GetHtml(new HttpItem
  583. // {
  584. // Url = url,
  585. // Method = method,
  586. // WebProxy = new WebProxy(CommonHelper.GetIp())
  587. // });
  588. // }
  589. //}
  590. return "";
  591. }
  592. #endregion
  593. #region 时间
  594. /// <summary>
  595. /// 将c# DateTime时间格式转换为Unix时间戳格式
  596. /// </summary>
  597. /// <param name="time">时间</param>
  598. /// <returns>long</returns>
  599. public static long ConvertDateTimeToInt(System.DateTime time)
  600. {
  601. System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0));
  602. long t = (time.Ticks - startTime.Ticks) / 10000; //除10000调整为13位
  603. return t;
  604. }
  605. /// <summary>
  606. /// 时间戳转时间
  607. /// </summary>
  608. /// <param name="unixTimeStamp"></param>
  609. /// <returns></returns>
  610. public static DateTime ConvertIntToDateTime(string unixTimeStamp)
  611. {
  612. DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
  613. long lTime = long.Parse(unixTimeStamp + "0000");
  614. TimeSpan toNow = new TimeSpan(lTime);
  615. DateTime targetDt = dtStart.Add(toNow);
  616. return dtStart.Add(toNow);
  617. }
  618. #endregion
  619. /// <summary>
  620. /// 线程是否执行完成
  621. /// </summary>
  622. /// <returns></returns>
  623. public static bool ThreadsFinsh()
  624. {
  625. int maxWorkerThreads, workerThreads;
  626. int maxportThreads, portThreads;
  627. /*
  628. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  629. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  630. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  631. */
  632. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  633. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  634. Thread.Sleep(3000);
  635. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
  636. if (maxWorkerThreads - workerThreads == 0)
  637. {
  638. Trace.WriteLine("加载完成!");
  639. return true;
  640. }
  641. return false;
  642. }
  643. /// <summary>
  644. /// 线程是否执行完成
  645. /// </summary>
  646. /// <returns></returns>
  647. public static bool ThreadsFinsh_new()
  648. {
  649. int maxWorkerThreads, workerThreads;
  650. int maxportThreads, portThreads;
  651. /*
  652. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  653. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  654. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  655. */
  656. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  657. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  658. Thread.Sleep(3000);
  659. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads - 3));
  660. if (maxWorkerThreads - workerThreads - 3 == 0)
  661. {
  662. Trace.WriteLine("加载完成!");
  663. return true;
  664. }
  665. return false;
  666. }
  667. /// <summary>
  668. /// 文件写入
  669. /// </summary>
  670. public static void Write(string path, string data)
  671. {
  672. using (var fs = new FileStream(path, FileMode.Append))
  673. {
  674. using (var sw = new StreamWriter(fs))
  675. {
  676. sw.WriteLine(data);
  677. sw.Flush();
  678. }
  679. }
  680. }
  681. public static void Write_IP(string path, string data)
  682. {
  683. using (var fs = new FileStream(path, FileMode.Create))
  684. {
  685. using (var sw = new StreamWriter(fs, Encoding.UTF8))
  686. {
  687. sw.Write(data);
  688. sw.Flush();
  689. }
  690. }
  691. }
  692. public static void LogBD(string content, string pathName = "",string directoryName="Log")
  693. {
  694. if (pathName.IsEmpty())
  695. pathName = content;
  696. var path = AppDomain.CurrentDomain.BaseDirectory + "/"+ directoryName;
  697. CreateDirectory(path);
  698. path += $"/{DateTime.Now.ToString("yyyyMMdd")}";
  699. CreateDirectory(path);
  700. Write(path + $"/{pathName}.txt", content + "||" + DateTime.Now.ToString());
  701. }
  702. /// <summary>
  703. /// 创建文件夹
  704. /// </summary>
  705. /// <param name="paht"></param>
  706. private static void CreateDirectory(string path)
  707. {
  708. if (!Directory.Exists(path))
  709. Directory.CreateDirectory(path);
  710. }
  711. public static T Mapper<T>(object data)
  712. {
  713. return AutoMapper.Mapper.DynamicMap<T>(data);
  714. }
  715. }
  716. class AsyncSemaphore
  717. {
  718. private readonly static Task s_completed = Task.FromResult(true);
  719. private readonly Queue<TaskCompletionSource<bool>> m_waiters = new Queue<TaskCompletionSource<bool>>();
  720. private int m_currentCount;
  721. public AsyncSemaphore(int initialCount)
  722. {
  723. if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount");
  724. m_currentCount = initialCount;
  725. }
  726. public Task WaitAsync()
  727. {
  728. lock (m_waiters)
  729. {
  730. if (m_currentCount > 0)
  731. {
  732. --m_currentCount;
  733. return s_completed;
  734. }
  735. else
  736. {
  737. var waiter = new TaskCompletionSource<bool>();
  738. m_waiters.Enqueue(waiter);
  739. return waiter.Task;
  740. }
  741. }
  742. }
  743. public void Release()
  744. {
  745. TaskCompletionSource<bool> toRelease = null;
  746. lock (m_waiters)
  747. {
  748. if (m_waiters.Count > 0)
  749. toRelease = m_waiters.Dequeue();
  750. else
  751. ++m_currentCount;
  752. }
  753. if (toRelease != null)
  754. toRelease.SetResult(true);
  755. }
  756. }
  757. public class AsyncLock
  758. {
  759. private readonly AsyncSemaphore m_semaphore;
  760. private readonly Task<Releaser> m_releaser;
  761. public AsyncLock()
  762. {
  763. m_semaphore = new AsyncSemaphore(1);
  764. m_releaser = Task.FromResult(new Releaser(this));
  765. }
  766. public Task<Releaser> LockAsync()
  767. {
  768. var wait = m_semaphore.WaitAsync();
  769. return wait.IsCompleted ?
  770. m_releaser :
  771. wait.ContinueWith((_, state) => new Releaser((AsyncLock)state),
  772. this, CancellationToken.None,
  773. TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
  774. }
  775. public struct Releaser : IDisposable
  776. {
  777. private readonly AsyncLock m_toRelease;
  778. internal Releaser(AsyncLock toRelease) { m_toRelease = toRelease; }
  779. public void Dispose()
  780. {
  781. if (m_toRelease != null)
  782. m_toRelease.m_semaphore.Release();
  783. }
  784. }
  785. }
  786. }