CommonHelper.cs 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859
  1. using FCS.Models;
  2. using FCS.Models.DTO;
  3. using HtmlAgilityPack;
  4. using Newtonsoft.Json;
  5. using Quartz;
  6. using System;
  7. using System.Collections.Generic;
  8. using System.Configuration;
  9. using System.Diagnostics;
  10. using System.IO;
  11. using System.Linq;
  12. using System.Net;
  13. using System.Reflection;
  14. using System.Runtime.Remoting.Messaging;
  15. using System.Text;
  16. using System.Text.RegularExpressions;
  17. using System.Threading;
  18. using System.Threading.Tasks;
  19. using System.Xml;
  20. namespace FCS.Common
  21. {
  22. /// <summary>
  23. /// 公用帮助类
  24. /// </summary>
  25. public static class CommonHelper
  26. {
  27. /// <summary>
  28. /// 将XML内容转换成目标对象实体集合
  29. /// </summary>
  30. /// <typeparam name="T">目标对象实体</typeparam>
  31. /// <param name="FileName">完整文件名(根目录下只需文件名称)</param>
  32. /// <param name="WrapperNodeName"></param>
  33. /// <returns></returns>
  34. public static List<T> ConvertXMLToObject<T>(string FileName, string WrapperNodeName)
  35. {
  36. XmlDocument doc = new XmlDocument();
  37. doc.Load(FileName);
  38. List<T> result = new List<T>();
  39. var TType = typeof(T);
  40. XmlNodeList nodeList = doc.ChildNodes;
  41. if (!string.IsNullOrEmpty(WrapperNodeName))
  42. {
  43. foreach (XmlNode node in doc.ChildNodes)
  44. {
  45. if (node.Name == WrapperNodeName)
  46. {
  47. nodeList = node.ChildNodes;
  48. break;
  49. }
  50. }
  51. }
  52. object oneT = null;
  53. foreach (XmlNode node in nodeList)
  54. {
  55. if (node.NodeType == XmlNodeType.Comment || node.NodeType == XmlNodeType.XmlDeclaration) continue;
  56. oneT = TType.Assembly.CreateInstance(TType.FullName);
  57. foreach (XmlNode item in node.ChildNodes)
  58. {
  59. if (item.NodeType == XmlNodeType.Comment) continue;
  60. var property = TType.GetProperty(item.Name);
  61. if (property != null)
  62. property.SetValue(oneT, Convert.ChangeType(item.InnerText, property.PropertyType), null);
  63. }
  64. result.Add((T)oneT);
  65. }
  66. return result;
  67. }
  68. /// <summary>
  69. /// 从作业数据地图中获取配置信息
  70. /// </summary>
  71. /// <param name="datamap">作业数据地图</param>
  72. /// <returns></returns>
  73. public static FCSConfig GetConfigFromDataMap(JobDataMap datamap)
  74. {
  75. FCSConfig config = new FCSConfig();
  76. var properties = typeof(FCSConfig).GetProperties();
  77. foreach (PropertyInfo info in properties)
  78. {
  79. if (info.PropertyType == typeof(string))
  80. info.SetValue(config, datamap.GetString(info.Name), null);
  81. else if (info.PropertyType == typeof(Int32))
  82. info.SetValue(config, datamap.GetInt(info.Name), null);
  83. }
  84. return config;
  85. }
  86. #region 日志信息
  87. public static string GetJobMainLogInfo(string QiHao)
  88. {
  89. return string.Format("通过主站地址抓取{0}期开奖数据成功", QiHao);
  90. }
  91. public static string GetJobLogError(string QiHao)
  92. {
  93. return string.Format("【{0}】抓取期开奖数据失败", QiHao);
  94. }
  95. #endregion 日志信息
  96. /// <summary>
  97. /// 将值转换为T类型数据
  98. /// </summary>
  99. /// <typeparam name="T">目标类型</typeparam>
  100. /// <param name="value">数据值</param>
  101. /// <returns></returns>
  102. public static T ChangeType<T>(object value)
  103. {
  104. return ChangeType<T>(value, default(T));
  105. }
  106. /// <summary>
  107. /// 将值转换为T类型数据,失败则返回T类型默认值
  108. /// </summary>
  109. /// <typeparam name="T">目标类型</typeparam>
  110. /// <param name="value">数据值</param>
  111. /// <param name="defaultValue">T类型默认值</param>
  112. /// <returns></returns>
  113. public static T ChangeType<T>(object value, T defaultValue)
  114. {
  115. if (value != null)
  116. {
  117. Type nullableType = typeof(T);
  118. if (!nullableType.IsInterface && (!nullableType.IsClass || (nullableType == typeof(string))))
  119. {
  120. if (nullableType.IsGenericType && (nullableType.GetGenericTypeDefinition() == typeof(Nullable<>)))
  121. {
  122. return (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(nullableType));
  123. }
  124. if (nullableType.IsEnum)
  125. {
  126. return (T)Enum.Parse(nullableType, value.ToString());
  127. }
  128. return (T)Convert.ChangeType(value, nullableType);
  129. }
  130. if (value is T)
  131. {
  132. return (T)value;
  133. }
  134. }
  135. return defaultValue;
  136. }
  137. /// <summary>
  138. /// 将值转换为type类型的值
  139. /// </summary>
  140. /// <param name="value">值</param>
  141. /// <param name="type">目标类型</param>
  142. /// <returns></returns>
  143. public static object ChangeType(object value, Type type)
  144. {
  145. if (value != null)
  146. {
  147. var nullableType = Nullable.GetUnderlyingType(type);
  148. if (nullableType != null)//可空
  149. {
  150. return Convert.ChangeType(value, nullableType);
  151. }
  152. if (Convert.IsDBNull(value))//特殊处理,由于数据库类型与项目中的类型定义不匹配
  153. return type.IsValueType ? Activator.CreateInstance(type) : null;
  154. return Convert.ChangeType(value, type);
  155. }
  156. return null;
  157. }
  158. #region 获取全局唯一GUID
  159. /// <summary>
  160. /// 获取全局唯一GUID
  161. /// </summary>
  162. /// <param name="needReplace">是否需要替换-</param>
  163. /// <param name="format">格式化</param>
  164. /// <example>N:38bddf48f43c48588e0d78761eaa1ce6</example>>
  165. /// <example>P:(778406c2-efff-4262-ab03-70a77d09c2b5)</example>>
  166. /// <example>B:{09f140d5-af72-44ba-a763-c861304b46f8}</example>>
  167. /// <example>D:57d99d89-caab-482a-a0e9-a0a803eed3ba</example>>
  168. /// <returns></returns>
  169. public static string GetGuid(bool needReplace = true, string format = "N")
  170. {
  171. Guid res = NewSequentialGuid();//Guid.NewGuid();
  172. return needReplace ? res.ToString(format) : res.ToString();
  173. }
  174. [System.Runtime.InteropServices.DllImport("rpcrt4.dll", SetLastError = true)]
  175. static extern int UuidCreateSequential(byte[] buffer);
  176. /// <summary>
  177. /// 创建有序GUID
  178. /// </summary>
  179. /// <returns></returns>
  180. private static Guid NewSequentialGuid()
  181. {
  182. byte[] raw = new byte[16];
  183. if (UuidCreateSequential(raw) != 0)
  184. throw new System.ComponentModel.Win32Exception(System.Runtime.InteropServices.Marshal.GetLastWin32Error());
  185. byte[] fix = new byte[16];
  186. // reverse 0..3
  187. fix[0x0] = raw[0x3];
  188. fix[0x1] = raw[0x2];
  189. fix[0x2] = raw[0x1];
  190. fix[0x3] = raw[0x0];
  191. // reverse 4 & 5
  192. fix[0x4] = raw[0x5];
  193. fix[0x5] = raw[0x4];
  194. // reverse 6 & 7
  195. fix[0x6] = raw[0x7];
  196. fix[0x7] = raw[0x6];
  197. // all other are unchanged
  198. fix[0x8] = raw[0x8];
  199. fix[0x9] = raw[0x9];
  200. fix[0xA] = raw[0xA];
  201. fix[0xB] = raw[0xB];
  202. fix[0xC] = raw[0xC];
  203. fix[0xD] = raw[0xD];
  204. fix[0xE] = raw[0xE];
  205. fix[0xF] = raw[0xF];
  206. return new Guid(fix);
  207. }
  208. #endregion 获取全局唯一GUID
  209. #region HtmlAgilityPack
  210. public static int IpCount;
  211. public static List<string> IpList;
  212. public static string GetIp(string path = "", bool isGetIp = true)
  213. {
  214. if (isGetIp && IpList.Count > 0)
  215. {
  216. var ran = new Random().Next(0, IpList.Count);
  217. return IpList[ran];
  218. }
  219. if (path.IsEmpty())
  220. path = AppDomain.CurrentDomain.BaseDirectory + "/XmlConfig/IP.txt";
  221. StreamReader sr;
  222. try
  223. {
  224. sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
  225. }
  226. catch (Exception)
  227. {
  228. Thread.Sleep(1000);
  229. sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
  230. }
  231. string content = sr.ReadToEnd().ToString();
  232. sr.Close();
  233. var list = content.JsonToList<string>();
  234. if (isGetIp)
  235. IpList = list;
  236. var ram = new Random().Next(0, list.Count);
  237. IpCount = list.Count;
  238. return list[ram];
  239. }
  240. static object locker = new object();
  241. static bool isGetIp = false;
  242. public delegate string GetIPDataBYOne(List<string> _urlList, string _title = "", bool isFormData = false);
  243. public delegate string GetIPDataBYOne_FormData(List<string> _urlList, Dictionary<string, string> formData, string _title = "");
  244. /// <summary>
  245. /// 获取HTML
  246. /// </summary>
  247. /// <param name="model">参数实体</param>
  248. /// <returns></returns>
  249. public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model)
  250. {
  251. var doc = new HtmlDocument();
  252. doc.LoadHtml(GetHtmlByIP(model));
  253. return doc;
  254. }
  255. /// <summary>
  256. /// 获取HTML
  257. /// </summary>
  258. /// <param name="model">参数实体</param>
  259. /// <returns></returns>
  260. public static string GetHtmlString(HtmlParameterDTO model)
  261. {
  262. return GetHtmlByIP(model);
  263. }
  264. /// <summary>
  265. /// 通过Ip获取页面的HTML
  266. /// </summary>
  267. /// <param name="model"></param>
  268. /// <returns></returns>
  269. private static string GetHtmlByIP(HtmlParameterDTO model)
  270. {
  271. var NotIpList = new List<string>();
  272. Stopwatch sw = new Stopwatch();
  273. sw.Start();
  274. var ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  275. var httpItem = Mapper<HttpItem>(model);
  276. httpItem.WebProxy = new WebProxy(ip);
  277. var html = new HttpHelper().GetHtml(httpItem);
  278. //对文本的检查
  279. while ((model.IsCheckEmpty && html.Html.IsEmpty())
  280. || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
  281. || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
  282. || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
  283. || html.Html.Contains("502 Bad Gateway")
  284. || html.Html.Contains("400 Bad Request")
  285. || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
  286. || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
  287. || html.Html.Contains("缓存访问被拒绝")
  288. || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title)))
  289. {
  290. if (html.Html.ToLower().Contains("exception report"))
  291. {
  292. return ConfigurationManager.AppSettings["Termination"].ToString();
  293. }
  294. NotIpList.Add(ip);
  295. if (NotIpList.Distinct().ToList().Count == IpCount || NotIpList.Distinct().ToList().Count > model.NotIpNumber)
  296. {
  297. //EmailHelper.Send("1625453870@qq.com", "IP用完,未获取值的URL", "URL:" + model.Url + "||参数:" + model.FormData.TryToJson());
  298. //return ConfigurationManager.AppSettings["Termination"].ToString();
  299. LogBD(model.Url, "LogUrl", "UrlLog");
  300. return ConfigurationManager.AppSettings["Termination"].ToString();
  301. //IAsyncResult asyncResult;
  302. //lock (locker)
  303. //{
  304. // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
  305. // asyncResult = task.BeginInvoke(new List<string> { model.Url }, model.Title, false, null, null);
  306. // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
  307. // {
  308. // }
  309. // ip = task.EndInvoke(asyncResult);
  310. // if (ip.IsEmpty())
  311. // return ConfigurationManager.AppSettings["Termination"].ToString();
  312. // else
  313. // return ip;
  314. //}
  315. }
  316. else
  317. {
  318. ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  319. while (NotIpList.Contains(ip) && model.IP.IsEmpty())
  320. ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
  321. }
  322. httpItem.WebProxy = new WebProxy(ip);
  323. html = new HttpHelper().GetHtml(httpItem);
  324. }
  325. sw.Stop();
  326. Trace.WriteLine("url:" + model.Url + "||IP:" + ip + "||时间:" + sw.ElapsedMilliseconds + "毫秒");
  327. return html.Html;
  328. }
  329. /// <summary>
  330. /// 得到HtmlDocument
  331. /// </summary>
  332. /// <param name="url"></param>
  333. /// <param name="method"></param>
  334. /// <returns></returns>
  335. public static HtmlDocument GetHtml(string url, string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get", int timeout = 90 * 1000, int notIpNUmber = 100)
  336. {
  337. return GetHtmlHtmlDocument(new HtmlParameterDTO
  338. {
  339. Url = url,
  340. Title = title,
  341. IP = webProxy,
  342. Method = method,
  343. Timeout = timeout,
  344. NotIpNumber = notIpNUmber
  345. });
  346. }
  347. /// <summary>
  348. /// 得到HtmlDocument
  349. /// From表单提交
  350. /// </summary>
  351. /// <param name="url"></param>
  352. /// <param name="method"></param>
  353. /// <returns></returns>
  354. public static HtmlDocument GetHtml(string url, Dictionary<string, string> formData, string title = "", string webProxy = "", int timeout = 90 * 1000, int notIpNUmber = 100)
  355. {
  356. // ContentType = "application/x-www-form-urlencoded",
  357. return GetHtmlHtmlDocument(new HtmlParameterDTO
  358. {
  359. Url = url,
  360. Title = title,
  361. IP = webProxy,
  362. Method = "POST",
  363. Timeout = timeout,
  364. NotIpNumber = notIpNUmber,
  365. ContentType = "application/x-www-form-urlencoded",
  366. FormData = formData
  367. });
  368. }
  369. #endregion
  370. #region lg
  371. /// <summary>
  372. /// 得到HtmlDocument
  373. /// </summary>
  374. /// <param name="url"></param>
  375. /// <param name="method"></param>
  376. /// <returns></returns>
  377. public static string GetHtmlString(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
  378. {
  379. var NotIpList = new List<string>();
  380. var ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  381. var html = new HttpHelper().GetHtml(new HttpItem
  382. {
  383. Url = url,
  384. Method = method,
  385. WebProxy = new WebProxy(ip),
  386. Timeout = timeout
  387. });
  388. int number = 0;
  389. if (!isWebSoxket)
  390. {
  391. while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  392. || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  393. || (!title.IsEmpty() && html.Html.IndexOf(title) == -1)
  394. || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
  395. || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
  396. || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
  397. || html.Html.Contains("502 Bad Gateway")
  398. || html.Html.Contains("400 Bad Request")
  399. || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
  400. || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
  401. || html.Html.Contains("缓存访问被拒绝")
  402. || (title.IsEmpty() && !html.Html.Contains(title)))
  403. {
  404. number++;
  405. if (number > 40)
  406. return "";
  407. if (html.Html.ToLower().Contains("exception report"))
  408. {
  409. ConfigurationManager.AppSettings["Termination"].ToString();
  410. break;
  411. }
  412. NotIpList.Add(ip);
  413. if (NotIpList.Distinct().ToList().Count == IpCount)
  414. {
  415. //IAsyncResult asyncResult;
  416. //lock (locker)
  417. //{
  418. // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
  419. // asyncResult = task.BeginInvoke(new List<string> { url }, title, false, null, null);
  420. // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
  421. // {
  422. // }
  423. // ip = task.EndInvoke(asyncResult);
  424. // if (ip.IsEmpty())
  425. // {
  426. return ConfigurationManager.AppSettings["Termination"].ToString();
  427. // break;
  428. // }
  429. //}
  430. //}
  431. }
  432. else
  433. ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  434. while (NotIpList.Contains(ip))
  435. ip = webProxy.IsEmpty() ? GetIp() : webProxy;
  436. html = new HttpHelper().GetHtml(new HttpItem
  437. {
  438. Url = url,
  439. Method = method,
  440. WebProxy = new WebProxy(ip)
  441. });
  442. }
  443. }
  444. return html.Html;
  445. }
  446. /// <summary>
  447. /// 得到HtmlDocument
  448. /// </summary>
  449. /// <param name="url"></param>
  450. /// <param name="method"></param>
  451. /// <returns></returns>
  452. public static HttpResult GetPostHtmlString(string url, HttpItem model, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
  453. {
  454. model.Timeout = timeout;
  455. var html = new HttpHelper().GetHtml(model);
  456. if (!isWebSoxket)
  457. {
  458. var number = 0;
  459. while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  460. || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  461. || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
  462. {
  463. number++;
  464. if (number > 20)
  465. return html;
  466. model.WebProxy = new WebProxy(GetIp());
  467. html = new HttpHelper().GetHtml(model);
  468. }
  469. }
  470. return html;
  471. }
  472. public static string GetHtmlString_ceshi(string url = "http://fenxi.zgzcw.com/2321966/bjop", string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get")
  473. {
  474. var html3 = new HttpHelper().GetHtml(new HttpItem
  475. {
  476. Url = url,
  477. Method = method,
  478. WebProxy = new WebProxy("113.124.93.1:601")
  479. });
  480. html3 = new HttpHelper().GetHtml(new HttpItem
  481. {
  482. Url = url,
  483. Method = method,
  484. WebProxy = new WebProxy("117.91.249.95:601")
  485. });
  486. F_Grouping g = new F_Grouping();
  487. List<string> list = new List<string>();
  488. list.Add("27.26.162.129");
  489. list.Add("117.91.249.95");
  490. list.Add("222.189.190.117");
  491. list.Add("222.189.191.254");
  492. list.Add("111.72.57.234");
  493. list.Add("113.124.93.1");
  494. list.Add("60.189.167.102");
  495. list.Add("180.118.141.126");
  496. list.Add("221.230.123.45");
  497. list.Add("221.230.123.101");
  498. list.Add("221.230.124.127");
  499. list.Add("125.125.45.149");
  500. list.Add("182.34.32.90");
  501. list.Add("113.121.46.19");
  502. list.Add("113.121.45.103");
  503. list.Add("113.121.23.219");
  504. list.Add("111.72.56.135");
  505. list.Add("111.72.63.12");
  506. list.Add("111.72.62.174");
  507. list.Add("111.72.58.28");
  508. list.Add("111.72.62.226");
  509. list.Add("106.226.227.245");
  510. list.Add("111.79.173.163");
  511. list.Add("106.7.78.39");
  512. list.Add("182.84.86.158");
  513. list.Add("182.100.238.11");
  514. list.Add("111.72.107.203");
  515. List<int> ss = new List<int>();
  516. for (int i = 0; i < 65535; i++)
  517. {
  518. ss.Add(i + 1);
  519. }
  520. int max1 = list.Count;
  521. int num1 = 0;
  522. list.ForEach(async p =>
  523. {
  524. await Task.Run(() =>
  525. {
  526. int max = 65535;
  527. int num = 0;
  528. //比赛
  529. ss.ForEach(async p1 =>
  530. {
  531. await Task.Run(() =>
  532. {
  533. var html = new HttpHelper().GetHtml(new HttpItem
  534. {
  535. Url = url,
  536. Method = method,
  537. WebProxy = new WebProxy(p + ":" + p1)
  538. });
  539. });
  540. lock (g)
  541. {
  542. num++;
  543. Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
  544. }
  545. });
  546. lock (g)
  547. {
  548. while (num < max)
  549. {
  550. Monitor.Wait(g);//等待
  551. }
  552. }
  553. });
  554. lock (g)
  555. {
  556. num1++;
  557. Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
  558. }
  559. });
  560. lock (g)
  561. {
  562. while (num1 < max1)
  563. {
  564. Monitor.Wait(g);//等待
  565. }
  566. }
  567. //if (!isWebSoxket)
  568. //{
  569. // var number = 0;
  570. // while (html.Html.IsEmpty() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
  571. // || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
  572. // || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
  573. // {
  574. // number++;
  575. // if (number > 100)
  576. // return null;
  577. // //if (number > IpCount)
  578. // html = new HttpHelper().GetHtml(new HttpItem
  579. // {
  580. // Url = url,
  581. // Method = method,
  582. // WebProxy = new WebProxy(CommonHelper.GetIp())
  583. // });
  584. // }
  585. //}
  586. return "";
  587. }
  588. #endregion
  589. #region 时间
  590. /// <summary>
  591. /// 将c# DateTime时间格式转换为Unix时间戳格式
  592. /// </summary>
  593. /// <param name="time">时间</param>
  594. /// <returns>long</returns>
  595. public static long ConvertDateTimeToInt(System.DateTime time)
  596. {
  597. System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0));
  598. long t = (time.Ticks - startTime.Ticks) / 10000; //除10000调整为13位
  599. return t;
  600. }
  601. /// <summary>
  602. /// 时间戳转时间
  603. /// </summary>
  604. /// <param name="unixTimeStamp"></param>
  605. /// <returns></returns>
  606. public static DateTime ConvertIntToDateTime(string unixTimeStamp)
  607. {
  608. DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
  609. long lTime = long.Parse(unixTimeStamp + "0000");
  610. TimeSpan toNow = new TimeSpan(lTime);
  611. DateTime targetDt = dtStart.Add(toNow);
  612. return dtStart.Add(toNow);
  613. }
  614. #endregion
  615. /// <summary>
  616. /// 线程是否执行完成
  617. /// </summary>
  618. /// <returns></returns>
  619. public static bool ThreadsFinsh()
  620. {
  621. int maxWorkerThreads, workerThreads;
  622. int maxportThreads, portThreads;
  623. /*
  624. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  625. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  626. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  627. */
  628. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  629. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  630. Thread.Sleep(3000);
  631. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
  632. if (maxWorkerThreads - workerThreads == 0)
  633. {
  634. Trace.WriteLine("加载完成!");
  635. return true;
  636. }
  637. return false;
  638. }
  639. /// <summary>
  640. /// 线程是否执行完成
  641. /// </summary>
  642. /// <returns></returns>
  643. public static bool ThreadsFinsh_new()
  644. {
  645. int maxWorkerThreads, workerThreads;
  646. int maxportThreads, portThreads;
  647. /*
  648. GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
  649. 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
  650. 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
  651. */
  652. ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
  653. ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
  654. Thread.Sleep(3000);
  655. Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads - 3));
  656. if (maxWorkerThreads - workerThreads - 3 == 0)
  657. {
  658. Trace.WriteLine("加载完成!");
  659. return true;
  660. }
  661. return false;
  662. }
  663. /// <summary>
  664. /// 文件写入
  665. /// </summary>
  666. public static void Write(string path, string data)
  667. {
  668. using (var fs = new FileStream(path, FileMode.Append))
  669. {
  670. using (var sw = new StreamWriter(fs))
  671. {
  672. sw.WriteLine(data);
  673. sw.Flush();
  674. }
  675. }
  676. }
  677. public static void Write_IP(string path, string data)
  678. {
  679. using (var fs = new FileStream(path, FileMode.Create))
  680. {
  681. using (var sw = new StreamWriter(fs, Encoding.UTF8))
  682. {
  683. sw.Write(data);
  684. sw.Flush();
  685. }
  686. }
  687. }
  688. public static void LogBD(string content, string pathName = "",string directoryName="Log")
  689. {
  690. if (pathName.IsEmpty())
  691. pathName = content;
  692. var path = AppDomain.CurrentDomain.BaseDirectory + "/"+ directoryName;
  693. CreateDirectory(path);
  694. path += $"/{DateTime.Now.ToString("yyyyMMdd")}";
  695. CreateDirectory(path);
  696. Write(path + $"/{pathName}.txt", content + "||" + DateTime.Now.ToString());
  697. }
  698. /// <summary>
  699. /// 创建文件夹
  700. /// </summary>
  701. /// <param name="paht"></param>
  702. private static void CreateDirectory(string path)
  703. {
  704. if (!Directory.Exists(path))
  705. Directory.CreateDirectory(path);
  706. }
  707. public static T Mapper<T>(object data)
  708. {
  709. return AutoMapper.Mapper.DynamicMap<T>(data);
  710. }
  711. }
  712. class AsyncSemaphore
  713. {
  714. private readonly static Task s_completed = Task.FromResult(true);
  715. private readonly Queue<TaskCompletionSource<bool>> m_waiters = new Queue<TaskCompletionSource<bool>>();
  716. private int m_currentCount;
  717. public AsyncSemaphore(int initialCount)
  718. {
  719. if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount");
  720. m_currentCount = initialCount;
  721. }
  722. public Task WaitAsync()
  723. {
  724. lock (m_waiters)
  725. {
  726. if (m_currentCount > 0)
  727. {
  728. --m_currentCount;
  729. return s_completed;
  730. }
  731. else
  732. {
  733. var waiter = new TaskCompletionSource<bool>();
  734. m_waiters.Enqueue(waiter);
  735. return waiter.Task;
  736. }
  737. }
  738. }
  739. public void Release()
  740. {
  741. TaskCompletionSource<bool> toRelease = null;
  742. lock (m_waiters)
  743. {
  744. if (m_waiters.Count > 0)
  745. toRelease = m_waiters.Dequeue();
  746. else
  747. ++m_currentCount;
  748. }
  749. if (toRelease != null)
  750. toRelease.SetResult(true);
  751. }
  752. }
  753. public class AsyncLock
  754. {
  755. private readonly AsyncSemaphore m_semaphore;
  756. private readonly Task<Releaser> m_releaser;
  757. public AsyncLock()
  758. {
  759. m_semaphore = new AsyncSemaphore(1);
  760. m_releaser = Task.FromResult(new Releaser(this));
  761. }
  762. public Task<Releaser> LockAsync()
  763. {
  764. var wait = m_semaphore.WaitAsync();
  765. return wait.IsCompleted ?
  766. m_releaser :
  767. wait.ContinueWith((_, state) => new Releaser((AsyncLock)state),
  768. this, CancellationToken.None,
  769. TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
  770. }
  771. public struct Releaser : IDisposable
  772. {
  773. private readonly AsyncLock m_toRelease;
  774. internal Releaser(AsyncLock toRelease) { m_toRelease = toRelease; }
  775. public void Dispose()
  776. {
  777. if (m_toRelease != null)
  778. m_toRelease.m_semaphore.Release();
  779. }
  780. }
  781. }
  782. }