CommonHelper.cs 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856
  1. using FCS.Models;
  2. using FCS.Models.DTO;
  3. using HtmlAgilityPack;
  4. using Newtonsoft.Json;
  5. using Quartz;
  6. using System;
  7. using System.Collections.Generic;
  8. using System.Configuration;
  9. using System.Diagnostics;
  10. using System.IO;
  11. using System.Linq;
  12. using System.Net;
  13. using System.Reflection;
  14. using System.Runtime.Remoting.Messaging;
  15. using System.Text;
  16. using System.Text.RegularExpressions;
  17. using System.Threading;
  18. using System.Threading.Tasks;
  19. using System.Xml;
  20. namespace FCS.Common
  21. {
  22. /// <summary>
  23. /// 公用帮助类
  24. /// </summary>
  25. public static class CommonHelper
  26. {
  27. /// <summary>
  28. /// 将XML内容转换成目标对象实体集合
  29. /// </summary>
  30. /// <typeparam name="T">目标对象实体</typeparam>
  31. /// <param name="FileName">完整文件名(根目录下只需文件名称)</param>
  32. /// <param name="WrapperNodeName"></param>
  33. /// <returns></returns>
  34. public static List<T> ConvertXMLToObject<T>(string FileName, string WrapperNodeName)
  35. {
  36. XmlDocument doc = new XmlDocument();
  37. doc.Load(FileName);
  38. List<T> result = new List<T>();
  39. var TType = typeof(T);
  40. XmlNodeList nodeList = doc.ChildNodes;
  41. if (!string.IsNullOrEmpty(WrapperNodeName))
  42. {
  43. foreach (XmlNode node in doc.ChildNodes)
  44. {
  45. if (node.Name == WrapperNodeName)
  46. {
  47. nodeList = node.ChildNodes;
  48. break;
  49. }
  50. }
  51. }
  52. object oneT = null;
  53. foreach (XmlNode node in nodeList)
  54. {
  55. if (node.NodeType == XmlNodeType.Comment || node.NodeType == XmlNodeType.XmlDeclaration) continue;
  56. oneT = TType.Assembly.CreateInstance(TType.FullName);
  57. foreach (XmlNode item in node.ChildNodes)
  58. {
  59. if (item.NodeType == XmlNodeType.Comment) continue;
  60. var property = TType.GetProperty(item.Name);
  61. if (property != null)
  62. property.SetValue(oneT, Convert.ChangeType(item.InnerText, property.PropertyType), null);
  63. }
  64. result.Add((T)oneT);
  65. }
  66. return result;
  67. }
  68. /// <summary>
  69. /// 从作业数据地图中获取配置信息
  70. /// </summary>
  71. /// <param name="datamap">作业数据地图</param>
  72. /// <returns></returns>
  73. public static FCSConfig GetConfigFromDataMap(JobDataMap datamap)
  74. {
  75. FCSConfig config = new FCSConfig();
  76. var properties = typeof(FCSConfig).GetProperties();
  77. foreach (PropertyInfo info in properties)
  78. {
  79. if (info.PropertyType == typeof(string))
  80. info.SetValue(config, datamap.GetString(info.Name), null);
  81. else if (info.PropertyType == typeof(Int32))
  82. info.SetValue(config, datamap.GetInt(info.Name), null);
  83. }
  84. return config;
  85. }
  86. #region 日志信息
  87. public static string GetJobMainLogInfo(string QiHao)
  88. {
  89. return string.Format("通过主站地址抓取{0}期开奖数据成功", QiHao);
  90. }
  91. public static string GetJobLogError(string QiHao)
  92. {
  93. return string.Format("【{0}】抓取期开奖数据失败", QiHao);
  94. }
  95. #endregion 日志信息
  96. /// <summary>
  97. /// 将值转换为T类型数据
  98. /// </summary>
  99. /// <typeparam name="T">目标类型</typeparam>
  100. /// <param name="value">数据值</param>
  101. /// <returns></returns>
  102. public static T ChangeType<T>(object value)
  103. {
  104. return ChangeType<T>(value, default(T));
  105. }
  106. /// <summary>
  107. /// 将值转换为T类型数据,失败则返回T类型默认值
  108. /// </summary>
  109. /// <typeparam name="T">目标类型</typeparam>
  110. /// <param name="value">数据值</param>
  111. /// <param name="defaultValue">T类型默认值</param>
  112. /// <returns></returns>
  113. public static T ChangeType<T>(object value, T defaultValue)
  114. {
  115. if (value != null)
  116. {
  117. Type nullableType = typeof(T);
  118. if (!nullableType.IsInterface && (!nullableType.IsClass || (nullableType == typeof(string))))
  119. {
  120. if (nullableType.IsGenericType && (nullableType.GetGenericTypeDefinition() == typeof(Nullable<>)))
  121. {
  122. return (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(nullableType));
  123. }
  124. if (nullableType.IsEnum)
  125. {
  126. return (T)Enum.Parse(nullableType, value.ToString());
  127. }
  128. return (T)Convert.ChangeType(value, nullableType);
  129. }
  130. if (value is T)
  131. {
  132. return (T)value;
  133. }
  134. }
  135. return defaultValue;
  136. }
  137. /// <summary>
  138. /// 将值转换为type类型的值
  139. /// </summary>
  140. /// <param name="value">值</param>
  141. /// <param name="type">目标类型</param>
  142. /// <returns></returns>
  143. public static object ChangeType(object value, Type type)
  144. {
  145. if (value != null)
  146. {
  147. var nullableType = Nullable.GetUnderlyingType(type);
  148. if (nullableType != null)//可空
  149. {
  150. return Convert.ChangeType(value, nullableType);
  151. }
  152. if (Convert.IsDBNull(value))//特殊处理,由于数据库类型与项目中的类型定义不匹配
  153. return type.IsValueType ? Activator.CreateInstance(type) : null;
  154. return Convert.ChangeType(value, type);
  155. }
  156. return null;
  157. }
  158. #region 获取全局唯一GUID
  159. /// <summary>
  160. /// 获取全局唯一GUID
  161. /// </summary>
  162. /// <param name="needReplace">是否需要替换-</param>
  163. /// <param name="format">格式化</param>
  164. /// <example>N:38bddf48f43c48588e0d78761eaa1ce6</example>>
  165. /// <example>P:(778406c2-efff-4262-ab03-70a77d09c2b5)</example>>
  166. /// <example>B:{09f140d5-af72-44ba-a763-c861304b46f8}</example>>
  167. /// <example>D:57d99d89-caab-482a-a0e9-a0a803eed3ba</example>>
  168. /// <returns></returns>
  169. public static string GetGuid(bool needReplace = true, string format = "N")
  170. {
  171. Guid res = NewSequentialGuid();//Guid.NewGuid();
  172. return needReplace ? res.ToString(format) : res.ToString();
  173. }
  174. [System.Runtime.InteropServices.DllImport("rpcrt4.dll", SetLastError = true)]
  175. static extern int UuidCreateSequential(byte[] buffer);
  176. /// <summary>
  177. /// 创建有序GUID
  178. /// </summary>
  179. /// <returns></returns>
  180. private static Guid NewSequentialGuid()
  181. {
  182. byte[] raw = new byte[16];
  183. if (UuidCreateSequential(raw) != 0)
  184. throw new System.ComponentModel.Win32Exception(System.Runtime.InteropServices.Marshal.GetLastWin32Error());
  185. byte[] fix = new byte[16];
  186. // reverse 0..3
  187. fix[0x0] = raw[0x3];
  188. fix[0x1] = raw[0x2];
  189. fix[0x2] = raw[0x1];
  190. fix[0x3] = raw[0x0];
  191. // reverse 4 & 5
  192. fix[0x4] = raw[0x5];
  193. fix[0x5] = raw[0x4];
  194. // reverse 6 & 7
  195. fix[0x6] = raw[0x7];
  196. fix[0x7] = raw[0x6];
  197. // all other are unchanged
  198. fix[0x8] = raw[0x8];
  199. fix[0x9] = raw[0x9];
  200. fix[0xA] = raw[0xA];
  201. fix[0xB] = raw[0xB];
  202. fix[0xC] = raw[0xC];
  203. fix[0xD] = raw[0xD];
  204. fix[0xE] = raw[0xE];
  205. fix[0xF] = raw[0xF];
  206. return new Guid(fix);
  207. }
  208. #endregion 获取全局唯一GUID
  209. #region HtmlAgilityPack
  210. public static int IpCount;
  211. public static List<string> IpList;
  212. public static string GetIp(string path = "", bool isGetIp = true)
  213. {
  214. if (isGetIp && IpList.Count > 0)
  215. {
  216. var ran = new Random().Next(0, IpList.Count);
  217. return IpList[ran];
  218. }
  219. if (path.IsEmpty())
  220. path = AppDomain.CurrentDomain.BaseDirectory + "/XmlConfig/IP.txt";
  221. StreamReader sr;
  222. try
  223. {
  224. sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
  225. }
  226. catch (Exception)
  227. {
  228. Thread.Sleep(1000);
  229. sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
  230. }
  231. string content = sr.ReadToEnd().ToString();
  232. sr.Close();
  233. var list = content.JsonToList<string>();
  234. if (isGetIp)
  235. IpList = list;
  236. var ram = new Random().Next(0, list.Count);
  237. IpCount = list.Count;
  238. return list[ram];
  239. }
  240. static object locker = new object();
  241. static bool isGetIp = false;
  242. public delegate string GetIPDataBYOne(List<string> _urlList, string _title = "", bool isFormData = false);
  243. public delegate string GetIPDataBYOne_FormData(List<string> _urlList, Dictionary<string, string> formData, string _title = "");
  244. /// <summary>
  245. /// 获取HTML
  246. /// </summary>
  247. /// <param name="model">参数实体</param>
  248. /// <returns></returns>
  249. public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model)
  250. {
  251. var doc = new HtmlDocument();
  252. doc.LoadHtml(GetHtmlByIP(model));
  253. return doc;
  254. }
  255. /// <summary>
  256. /// 获取HTML
  257. /// </summary>
  258. /// <param name="model">参数实体</param>
  259. /// <returns></returns>
  260. public static string GetHtmlString(HtmlParameterDTO model)
  261. {
  262. return GetHtmlByIP(model);
  263. }
  264. /// <summary>
  265. /// 通过Ip获取页面的HTML
  266. /// </summary>
  267. /// <param name="model"></param>
  268. /// <returns></returns>
  269. private static string GetHtmlByIP(HtmlParameterDTO model)
  270. {
  271. var NotIpList = new List<string>();
  272. Stopwatch sw = new Stopwatch();
  273. sw.Start();
  274. var ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  275. var httpItem = Mapper<HttpItem>(model);
  276. httpItem.WebProxy = new WebProxy(ip);
  277. var html = new HttpHelper().GetHtml(httpItem);
  278. //对文本的检查
  279. while ((model.IsCheckEmpty && html.Html.IsEmpty())
  280. || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
  281. || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
  282. || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
  283. || html.Html.Contains("502 Bad Gateway")
  284. || html.Html.Contains("400 Bad Request")
  285. || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
  286. || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
  287. || html.Html.Contains("缓存访问被拒绝")
  288. || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title)))
  289. {
  290. if (html.Html.ToLower().Contains("exception report"))
  291. {
  292. return ConfigurationManager.AppSettings["Termination"].ToString();
  293. }
  294. NotIpList.Add(ip);
  295. if (NotIpList.Distinct().ToList().Count == IpCount || NotIpList.Distinct().ToList().Count > model.NotIpNumber)
  296. {
  297. EmailHelper.Send("1625453870@qq.com", "IP用完,未获取值的URL", "URL:" + model.Url + "||参数:" + model.FormData.TryToJson());
  298. return ConfigurationManager.AppSettings["Termination"].ToString();
  299. //IAsyncResult asyncResult;
  300. //lock (locker)
  301. //{
  302. // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
  303. // asyncResult = task.BeginInvoke(new List<string> { model.Url }, model.Title, false, null, null);
  304. // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
  305. // {
  306. // }
  307. // ip = task.EndInvoke(asyncResult);
  308. // if (ip.IsEmpty())
  309. // return ConfigurationManager.AppSettings["Termination"].ToString();
  310. // else
  311. // return ip;
  312. //}
  313. }
  314. else
  315. {
  316. ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  317. while (NotIpList.Contains(ip) && model.IP.IsEmpty())
  318. ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  319. }
  320. httpItem.WebProxy = new WebProxy(ip);
  321. html = new HttpHelper().GetHtml(httpItem);
  322. }
  323. sw.Stop();
  324. Trace.WriteLine("url:" + model.Url + "||IP:" + ip + "||时间:" + sw.ElapsedMilliseconds + "毫秒");
  325. return html.Html;
  326. }
  327. /// <summary>
  328. /// 得到HtmlDocument
  329. /// </summary>
  330. /// <param name="url"></param>
  331. /// <param name="method"></param>
  332. /// <returns></returns>
  333. public static HtmlDocument GetHtml(string url, string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get", int timeout = 90 * 1000, int notIpNUmber = 100)
  334. {
  335. return GetHtmlHtmlDocument(new HtmlParameterDTO
  336. {
  337. Url = url,
  338. Title = title,
  339. IP = webProxy,
  340. Method = method,
  341. Timeout = timeout,
  342. NotIpNumber = notIpNUmber
  343. });
  344. }
  345. /// <summary>
  346. /// 得到HtmlDocument
  347. /// From表单提交
  348. /// </summary>
  349. /// <param name="url"></param>
  350. /// <param name="method"></param>
  351. /// <returns></returns>
  352. public static HtmlDocument GetHtml(string url, Dictionary<string, string> formData, string title = "", string webProxy = "", int timeout = 90 * 1000, int notIpNUmber = 100)
  353. {
  354. // ContentType = "application/x-www-form-urlencoded",
  355. return GetHtmlHtmlDocument(new HtmlParameterDTO
  356. {
  357. Url = url,
  358. Title = title,
  359. IP = webProxy,
  360. Method = "POST",
  361. Timeout = timeout,
  362. NotIpNumber = notIpNUmber,
  363. ContentType = "application/x-www-form-urlencoded",
  364. FormData = formData
  365. });
  366. }
  367. #endregion
  368. #region lg
  369. /// <summary>
  370. /// 得到HtmlDocument
  371. /// </summary>
  372. /// <param name="url"></param>
  373. /// <param name="method"></param>
  374. /// <returns></returns>
  375. public static string GetHtmlString(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
  376. {
  377. var NotIpList = new List<string>();
  378. var ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  379. var html = new HttpHelper().GetHtml(new HttpItem
  380. {
  381. Url = url,
  382. Method = method,
  383. WebProxy = new WebProxy(ip),
  384. Timeout = timeout
  385. });
  386. int number = 0;
  387. if (!isWebSoxket)
  388. {
  389. while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  390. || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  391. || (!title.IsEmpty() && html.Html.IndexOf(title) == -1)
  392. || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
  393. || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
  394. || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
  395. || html.Html.Contains("502 Bad Gateway")
  396. || html.Html.Contains("400 Bad Request")
  397. || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
  398. || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
  399. || html.Html.Contains("缓存访问被拒绝")
  400. || (title.IsEmpty() && !html.Html.Contains(title)))
  401. {
  402. number++;
  403. if (number > 40)
  404. return "";
  405. if (html.Html.ToLower().Contains("exception report"))
  406. {
  407. ConfigurationManager.AppSettings["Termination"].ToString();
  408. break;
  409. }
  410. NotIpList.Add(ip);
  411. if (NotIpList.Distinct().ToList().Count == IpCount)
  412. {
  413. //IAsyncResult asyncResult;
  414. //lock (locker)
  415. //{
  416. // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
  417. // asyncResult = task.BeginInvoke(new List<string> { url }, title, false, null, null);
  418. // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
  419. // {
  420. // }
  421. // ip = task.EndInvoke(asyncResult);
  422. // if (ip.IsEmpty())
  423. // {
  424. return ConfigurationManager.AppSettings["Termination"].ToString();
  425. // break;
  426. // }
  427. //}
  428. //}
  429. }
  430. else
  431. ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  432. while (NotIpList.Contains(ip))
  433. ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  434. html = new HttpHelper().GetHtml(new HttpItem
  435. {
  436. Url = url,
  437. Method = method,
  438. WebProxy = new WebProxy(ip)
  439. });
  440. }
  441. }
  442. return html.Html;
  443. }
  444. /// <summary>
  445. /// 得到HtmlDocument
  446. /// </summary>
  447. /// <param name="url"></param>
  448. /// <param name="method"></param>
  449. /// <returns></returns>
  450. public static HttpResult GetPostHtmlString(string url, HttpItem model, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
  451. {
  452. model.Timeout = timeout;
  453. var html = new HttpHelper().GetHtml(model);
  454. if (!isWebSoxket)
  455. {
  456. var number = 0;
  457. while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  458. || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  459. || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
  460. {
  461. number++;
  462. if (number > 20)
  463. return html;
  464. model.WebProxy = new WebProxy(GetIp());
  465. html = new HttpHelper().GetHtml(model);
  466. }
  467. }
  468. return html;
  469. }
  470. public static string GetHtmlString_ceshi(string url = "http://fenxi.zgzcw.com/2321966/bjop", string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get")
  471. {
  472. var html3 = new HttpHelper().GetHtml(new HttpItem
  473. {
  474. Url = url,
  475. Method = method,
  476. WebProxy = new WebProxy("113.124.93.1:601")
  477. });
  478. html3 = new HttpHelper().GetHtml(new HttpItem
  479. {
  480. Url = url,
  481. Method = method,
  482. WebProxy = new WebProxy("117.91.249.95:601")
  483. });
  484. F_Grouping g = new F_Grouping();
  485. List<string> list = new List<string>();
  486. list.Add("27.26.162.129");
  487. list.Add("117.91.249.95");
  488. list.Add("222.189.190.117");
  489. list.Add("222.189.191.254");
  490. list.Add("111.72.57.234");
  491. list.Add("113.124.93.1");
  492. list.Add("60.189.167.102");
  493. list.Add("180.118.141.126");
  494. list.Add("221.230.123.45");
  495. list.Add("221.230.123.101");
  496. list.Add("221.230.124.127");
  497. list.Add("125.125.45.149");
  498. list.Add("182.34.32.90");
  499. list.Add("113.121.46.19");
  500. list.Add("113.121.45.103");
  501. list.Add("113.121.23.219");
  502. list.Add("111.72.56.135");
  503. list.Add("111.72.63.12");
  504. list.Add("111.72.62.174");
  505. list.Add("111.72.58.28");
  506. list.Add("111.72.62.226");
  507. list.Add("106.226.227.245");
  508. list.Add("111.79.173.163");
  509. list.Add("106.7.78.39");
  510. list.Add("182.84.86.158");
  511. list.Add("182.100.238.11");
  512. list.Add("111.72.107.203");
  513. List<int> ss = new List<int>();
  514. for (int i = 0; i < 65535; i++)
  515. {
  516. ss.Add(i + 1);
  517. }
  518. int max1 = list.Count;
  519. int num1 = 0;
  520. list.ForEach(async p =>
  521. {
  522. await Task.Run(() =>
  523. {
  524. int max = 65535;
  525. int num = 0;
  526. //比赛
  527. ss.ForEach(async p1 =>
  528. {
  529. await Task.Run(() =>
  530. {
  531. var html = new HttpHelper().GetHtml(new HttpItem
  532. {
  533. Url = url,
  534. Method = method,
  535. WebProxy = new WebProxy(p + ":" + p1)
  536. });
  537. });
  538. lock (g)
  539. {
  540. num++;
  541. Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
  542. }
  543. });
  544. lock (g)
  545. {
  546. while (num < max)
  547. {
  548. Monitor.Wait(g);//等待
  549. }
  550. }
  551. });
  552. lock (g)
  553. {
  554. num1++;
  555. Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
  556. }
  557. });
  558. lock (g)
  559. {
  560. while (num1 < max1)
  561. {
  562. Monitor.Wait(g);//等待
  563. }
  564. }
  565. //if (!isWebSoxket)
  566. //{
  567. // var number = 0;
  568. // while (html.Html.IsEmpty() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  569. // || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  570. // || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
  571. // {
  572. // number++;
  573. // if (number > 100)
  574. // return null;
  575. // //if (number > IpCount)
  576. // html = new HttpHelper().GetHtml(new HttpItem
  577. // {
  578. // Url = url,
  579. // Method = method,
  580. // WebProxy = new WebProxy(CommonHelper.GetIp())
  581. // });
  582. // }
  583. //}
  584. return "";
  585. }
  586. #endregion
  587. #region 时间
  588. /// <summary>
  589. /// 将c# DateTime时间格式转换为Unix时间戳格式
  590. /// </summary>
  591. /// <param name="time">时间</param>
  592. /// <returns>long</returns>
  593. public static long ConvertDateTimeToInt(System.DateTime time)
  594. {
  595. System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0));
  596. long t = (time.Ticks - startTime.Ticks) / 10000; //除10000调整为13位
  597. return t;
  598. }
  599. /// <summary>
  600. /// 时间戳转时间
  601. /// </summary>
  602. /// <param name="unixTimeStamp"></param>
  603. /// <returns></returns>
  604. public static DateTime ConvertIntToDateTime(string unixTimeStamp)
  605. {
  606. DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
  607. long lTime = long.Parse(unixTimeStamp + "0000");
  608. TimeSpan toNow = new TimeSpan(lTime);
  609. DateTime targetDt = dtStart.Add(toNow);
  610. return dtStart.Add(toNow);
  611. }
  612. #endregion
  613. /// <summary>
  614. /// 线程是否执行完成
  615. /// </summary>
  616. /// <returns></returns>
  617. public static bool ThreadsFinsh()
  618. {
  619. int maxWorkerThreads, workerThreads;
  620. int maxportThreads, portThreads;
  621. /*
  622. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  623. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  624. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  625. */
  626. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  627. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  628. Thread.Sleep(3000);
  629. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
  630. if (maxWorkerThreads - workerThreads == 0)
  631. {
  632. Trace.WriteLine("加载完成!");
  633. return true;
  634. }
  635. return false;
  636. }
  637. /// <summary>
  638. /// 线程是否执行完成
  639. /// </summary>
  640. /// <returns></returns>
  641. public static bool ThreadsFinsh_new()
  642. {
  643. int maxWorkerThreads, workerThreads;
  644. int maxportThreads, portThreads;
  645. /*
  646. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  647. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  648. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  649. */
  650. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  651. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  652. Thread.Sleep(3000);
  653. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads - 3));
  654. if (maxWorkerThreads - workerThreads - 3 == 0)
  655. {
  656. Trace.WriteLine("加载完成!");
  657. return true;
  658. }
  659. return false;
  660. }
  661. /// <summary>
  662. /// 文件写入
  663. /// </summary>
  664. public static void Write(string path, string data)
  665. {
  666. using (var fs = new FileStream(path, FileMode.Append))
  667. {
  668. using (var sw = new StreamWriter(fs))
  669. {
  670. sw.WriteLine(data);
  671. sw.Flush();
  672. }
  673. }
  674. }
  675. public static void Write_IP(string path, string data)
  676. {
  677. using (var fs = new FileStream(path, FileMode.Create))
  678. {
  679. using (var sw = new StreamWriter(fs, Encoding.UTF8))
  680. {
  681. sw.Write(data);
  682. sw.Flush();
  683. }
  684. }
  685. }
  686. public static void LogBD(string content, string pathName = "")
  687. {
  688. if (pathName.IsEmpty())
  689. pathName = content;
  690. var path = AppDomain.CurrentDomain.BaseDirectory + "/Log";
  691. CreateDirectory(path);
  692. path += $"/{DateTime.Now.ToString("yyyyMMdd")}";
  693. CreateDirectory(path);
  694. Write(path + $"/{pathName}.txt", content + "||" + DateTime.Now.ToString());
  695. }
  696. /// <summary>
  697. /// 创建文件夹
  698. /// </summary>
  699. /// <param name="paht"></param>
  700. private static void CreateDirectory(string path)
  701. {
  702. if (!Directory.Exists(path))
  703. Directory.CreateDirectory(path);
  704. }
  705. public static T Mapper<T>(object data)
  706. {
  707. return AutoMapper.Mapper.DynamicMap<T>(data);
  708. }
  709. }
  710. class AsyncSemaphore
  711. {
  712. private readonly static Task s_completed = Task.FromResult(true);
  713. private readonly Queue<TaskCompletionSource<bool>> m_waiters = new Queue<TaskCompletionSource<bool>>();
  714. private int m_currentCount;
  715. public AsyncSemaphore(int initialCount)
  716. {
  717. if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount");
  718. m_currentCount = initialCount;
  719. }
  720. public Task WaitAsync()
  721. {
  722. lock (m_waiters)
  723. {
  724. if (m_currentCount > 0)
  725. {
  726. --m_currentCount;
  727. return s_completed;
  728. }
  729. else
  730. {
  731. var waiter = new TaskCompletionSource<bool>();
  732. m_waiters.Enqueue(waiter);
  733. return waiter.Task;
  734. }
  735. }
  736. }
  737. public void Release()
  738. {
  739. TaskCompletionSource<bool> toRelease = null;
  740. lock (m_waiters)
  741. {
  742. if (m_waiters.Count > 0)
  743. toRelease = m_waiters.Dequeue();
  744. else
  745. ++m_currentCount;
  746. }
  747. if (toRelease != null)
  748. toRelease.SetResult(true);
  749. }
  750. }
  751. public class AsyncLock
  752. {
  753. private readonly AsyncSemaphore m_semaphore;
  754. private readonly Task<Releaser> m_releaser;
  755. public AsyncLock()
  756. {
  757. m_semaphore = new AsyncSemaphore(1);
  758. m_releaser = Task.FromResult(new Releaser(this));
  759. }
  760. public Task<Releaser> LockAsync()
  761. {
  762. var wait = m_semaphore.WaitAsync();
  763. return wait.IsCompleted ?
  764. m_releaser :
  765. wait.ContinueWith((_, state) => new Releaser((AsyncLock)state),
  766. this, CancellationToken.None,
  767. TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
  768. }
  769. public struct Releaser : IDisposable
  770. {
  771. private readonly AsyncLock m_toRelease;
  772. internal Releaser(AsyncLock toRelease) { m_toRelease = toRelease; }
  773. public void Dispose()
  774. {
  775. if (m_toRelease != null)
  776. m_toRelease.m_semaphore.Release();
  777. }
  778. }
  779. }
  780. }