using FCS.Models; using FCS.Models.DTO; using HtmlAgilityPack; using Newtonsoft.Json; using Quartz; using System; using System.Collections.Generic; using System.Configuration; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Reflection; using System.Runtime.Remoting.Messaging; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Xml; namespace FCS.Common { /// /// 公用帮助类 /// public static class CommonHelper { /// /// 将XML内容转换成目标对象实体集合 /// /// 目标对象实体 /// 完整文件名(根目录下只需文件名称) /// /// public static List ConvertXMLToObject(string FileName, string WrapperNodeName) { XmlDocument doc = new XmlDocument(); doc.Load(FileName); List result = new List(); var TType = typeof(T); XmlNodeList nodeList = doc.ChildNodes; if (!string.IsNullOrEmpty(WrapperNodeName)) { foreach (XmlNode node in doc.ChildNodes) { if (node.Name == WrapperNodeName) { nodeList = node.ChildNodes; break; } } } object oneT = null; foreach (XmlNode node in nodeList) { if (node.NodeType == XmlNodeType.Comment || node.NodeType == XmlNodeType.XmlDeclaration) continue; oneT = TType.Assembly.CreateInstance(TType.FullName); foreach (XmlNode item in node.ChildNodes) { if (item.NodeType == XmlNodeType.Comment) continue; var property = TType.GetProperty(item.Name); if (property != null) property.SetValue(oneT, Convert.ChangeType(item.InnerText, property.PropertyType), null); } result.Add((T)oneT); } return result; } /// /// 从作业数据地图中获取配置信息 /// /// 作业数据地图 /// public static FCSConfig GetConfigFromDataMap(JobDataMap datamap) { FCSConfig config = new FCSConfig(); var properties = typeof(FCSConfig).GetProperties(); foreach (PropertyInfo info in properties) { if (info.PropertyType == typeof(string)) info.SetValue(config, datamap.GetString(info.Name), null); else if (info.PropertyType == typeof(Int32)) info.SetValue(config, datamap.GetInt(info.Name), null); } return config; } #region 日志信息 public static string GetJobMainLogInfo(string QiHao) { return string.Format("通过主站地址抓取{0}期开奖数据成功", QiHao); } public static string GetJobLogError(string QiHao) { return string.Format("【{0}】抓取期开奖数据失败", QiHao); } #endregion 日志信息 /// /// 将值转换为T类型数据 /// /// 目标类型 /// 数据值 /// public static T ChangeType(object value) { return ChangeType(value, default(T)); } /// /// 将值转换为T类型数据,失败则返回T类型默认值 /// /// 目标类型 /// 数据值 /// T类型默认值 /// public static T ChangeType(object value, T defaultValue) { if (value != null) { Type nullableType = typeof(T); if (!nullableType.IsInterface && (!nullableType.IsClass || (nullableType == typeof(string)))) { if (nullableType.IsGenericType && (nullableType.GetGenericTypeDefinition() == typeof(Nullable<>))) { return (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(nullableType)); } if (nullableType.IsEnum) { return (T)Enum.Parse(nullableType, value.ToString()); } return (T)Convert.ChangeType(value, nullableType); } if (value is T) { return (T)value; } } return defaultValue; } /// /// 将值转换为type类型的值 /// /// 值 /// 目标类型 /// public static object ChangeType(object value, Type type) { if (value != null) { var nullableType = Nullable.GetUnderlyingType(type); if (nullableType != null)//可空 { return Convert.ChangeType(value, nullableType); } if (Convert.IsDBNull(value))//特殊处理,由于数据库类型与项目中的类型定义不匹配 return type.IsValueType ? Activator.CreateInstance(type) : null; return Convert.ChangeType(value, type); } return null; } #region 获取全局唯一GUID /// /// 获取全局唯一GUID /// /// 是否需要替换- /// 格式化 /// N:38bddf48f43c48588e0d78761eaa1ce6> /// P:(778406c2-efff-4262-ab03-70a77d09c2b5)> /// B:{09f140d5-af72-44ba-a763-c861304b46f8}> /// D:57d99d89-caab-482a-a0e9-a0a803eed3ba> /// public static string GetGuid(bool needReplace = true, string format = "N") { Guid res = NewSequentialGuid();//Guid.NewGuid(); return needReplace ? res.ToString(format) : res.ToString(); } [System.Runtime.InteropServices.DllImport("rpcrt4.dll", SetLastError = true)] static extern int UuidCreateSequential(byte[] buffer); /// /// 创建有序GUID /// /// private static Guid NewSequentialGuid() { byte[] raw = new byte[16]; if (UuidCreateSequential(raw) != 0) throw new System.ComponentModel.Win32Exception(System.Runtime.InteropServices.Marshal.GetLastWin32Error()); byte[] fix = new byte[16]; // reverse 0..3 fix[0x0] = raw[0x3]; fix[0x1] = raw[0x2]; fix[0x2] = raw[0x1]; fix[0x3] = raw[0x0]; // reverse 4 & 5 fix[0x4] = raw[0x5]; fix[0x5] = raw[0x4]; // reverse 6 & 7 fix[0x6] = raw[0x7]; fix[0x7] = raw[0x6]; // all other are unchanged fix[0x8] = raw[0x8]; fix[0x9] = raw[0x9]; fix[0xA] = raw[0xA]; fix[0xB] = raw[0xB]; fix[0xC] = raw[0xC]; fix[0xD] = raw[0xD]; fix[0xE] = raw[0xE]; fix[0xF] = raw[0xF]; return new Guid(fix); } #endregion 获取全局唯一GUID #region HtmlAgilityPack public static int IpCount; public static List IpList; public static string GetIp(string path = "", bool isGetIp = true) { if (isGetIp && IpList.Count > 0) { var ran = new Random().Next(0, IpList.Count); return IpList[ran]; } if (path.IsEmpty()) path = AppDomain.CurrentDomain.BaseDirectory + "/XmlConfig/IP.txt"; StreamReader sr; try { sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8")); } catch (Exception) { Thread.Sleep(1000); sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8")); } string content = sr.ReadToEnd().ToString(); sr.Close(); var list = content.JsonToList(); if (isGetIp) IpList = list; var ram = new Random().Next(0, list.Count); IpCount = list.Count; return list[ram]; } static object locker = new object(); static bool isGetIp = false; public delegate string GetIPDataBYOne(List _urlList, string _title = "", bool isFormData = false); public delegate string GetIPDataBYOne_FormData(List _urlList, Dictionary formData, string _title = ""); /// /// 获取HTML /// /// 参数实体 /// public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model) { var doc = new HtmlDocument(); doc.LoadHtml(GetHtmlByIP(model)); return doc; } /// /// 获取HTML /// /// 参数实体 /// public static string GetHtmlString(HtmlParameterDTO model) { return GetHtmlByIP(model); } /// /// 通过Ip获取页面的HTML /// /// /// private static string GetHtmlByIP(HtmlParameterDTO model) { var NotIpList = new List(); Stopwatch sw = new Stopwatch(); sw.Start(); var ip = model.IP.IsEmpty() ? GetIp() : model.IP; var httpItem = new HttpItem(); lock (locker) { httpItem=Mapper(model); httpItem.WebProxy = new WebProxy(ip); } var html = new HttpHelper().GetHtml(httpItem); //对文本的检查 while ((model.IsCheckEmpty && html.Html.IsEmpty()) || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden")) || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found")) || html.Html.Contains("502 Bad Gateway") || html.Html.Contains("400 Bad Request") || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently")) || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved")) || html.Html.Contains("缓存访问被拒绝") || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title))) { if (html.Html.ToLower().Contains("exception report")) { return ConfigurationManager.AppSettings["Termination"].ToString(); } NotIpList.Add(ip); if (NotIpList.Distinct().ToList().Count == IpCount || NotIpList.Distinct().ToList().Count > model.NotIpNumber) { //EmailHelper.Send("1625453870@qq.com", "IP用完,未获取值的URL", "URL:" + model.Url + "||参数:" + model.FormData.TryToJson()); //return ConfigurationManager.AppSettings["Termination"].ToString(); LogBD(model.Url, "LogUrl", "UrlLog"); return ConfigurationManager.AppSettings["Termination"].ToString(); //IAsyncResult asyncResult; //lock (locker) //{ // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne); // asyncResult = task.BeginInvoke(new List { model.Url }, model.Title, false, null, null); // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false)) // { // } // ip = task.EndInvoke(asyncResult); // if (ip.IsEmpty()) // return ConfigurationManager.AppSettings["Termination"].ToString(); // else // return ip; //} } else { ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP; while (NotIpList.Contains(ip) && model.IP.IsEmpty()) ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP; } httpItem.WebProxy = new WebProxy(ip); html = new HttpHelper().GetHtml(httpItem); } sw.Stop(); Trace.WriteLine("url:" + model.Url + "||IP:" + ip + "||时间:" + sw.ElapsedMilliseconds + "毫秒"); return html.Html; } /// /// 得到HtmlDocument /// /// /// /// public static HtmlDocument GetHtml(string url, string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get", int timeout = 90 * 1000, int notIpNUmber = 100) { return GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url, Title = title, IP = webProxy, Method = method, Timeout = timeout, NotIpNumber = notIpNUmber }); } /// /// 得到HtmlDocument /// From表单提交 /// /// /// /// public static HtmlDocument GetHtml(string url, Dictionary formData, string title = "", string webProxy = "", int timeout = 90 * 1000, int notIpNUmber = 100) { // ContentType = "application/x-www-form-urlencoded", return GetHtmlHtmlDocument(new HtmlParameterDTO { Url = url, Title = title, IP = webProxy, Method = "POST", Timeout = timeout, NotIpNumber = notIpNUmber, ContentType = "application/x-www-form-urlencoded", FormData = formData }); } #endregion #region lg /// /// 得到HtmlDocument /// /// /// /// public static string GetHtmlString(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100) { var NotIpList = new List(); var ip = webProxy.IsEmpty() ? GetIp() : webProxy; var html = new HttpHelper().GetHtml(new HttpItem { Url = url, Method = method, WebProxy = new WebProxy(ip), Timeout = timeout }); int number = 0; if (!isWebSoxket) { while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1) || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1) || (!title.IsEmpty() && html.Html.IndexOf(title) == -1) || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden")) || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found")) || html.Html.Contains("502 Bad Gateway") || html.Html.Contains("400 Bad Request") || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently")) || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved")) || html.Html.Contains("缓存访问被拒绝") || (title.IsEmpty() && !html.Html.Contains(title))) { number++; if (number > 40) return ""; if (html.Html.ToLower().Contains("exception report")) { ConfigurationManager.AppSettings["Termination"].ToString(); break; } NotIpList.Add(ip); if (NotIpList.Distinct().ToList().Count == IpCount) { //IAsyncResult asyncResult; //lock (locker) //{ // GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne); // asyncResult = task.BeginInvoke(new List { url }, title, false, null, null); // while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false)) // { // } // ip = task.EndInvoke(asyncResult); // if (ip.IsEmpty()) // { return ConfigurationManager.AppSettings["Termination"].ToString(); // break; // } //} //} } else ip = webProxy.IsEmpty() ? GetIp() : webProxy; while (NotIpList.Contains(ip)) ip = webProxy.IsEmpty() ? GetIp() : webProxy; html = new HttpHelper().GetHtml(new HttpItem { Url = url, Method = method, WebProxy = new WebProxy(ip) }); } } return html.Html; } /// /// 得到HtmlDocument /// /// /// /// public static HttpResult GetPostHtmlString(string url, HttpItem model, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100) { model.Timeout = timeout; var html = new HttpHelper().GetHtml(model); if (!isWebSoxket) { var number = 0; while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1) || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1) || (!title.IsEmpty() && html.Html.IndexOf(title) == -1)) { number++; if (number > 20) return html; model.WebProxy = new WebProxy(GetIp()); html = new HttpHelper().GetHtml(model); } } return html; } public static string GetHtmlString_ceshi(string url = "http://fenxi.zgzcw.com/2321966/bjop", string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get") { var html3 = new HttpHelper().GetHtml(new HttpItem { Url = url, Method = method, WebProxy = new WebProxy("113.124.93.1:601") }); html3 = new HttpHelper().GetHtml(new HttpItem { Url = url, Method = method, WebProxy = new WebProxy("117.91.249.95:601") }); F_Grouping g = new F_Grouping(); List list = new List(); list.Add("27.26.162.129"); list.Add("117.91.249.95"); list.Add("222.189.190.117"); list.Add("222.189.191.254"); list.Add("111.72.57.234"); list.Add("113.124.93.1"); list.Add("60.189.167.102"); list.Add("180.118.141.126"); list.Add("221.230.123.45"); list.Add("221.230.123.101"); list.Add("221.230.124.127"); list.Add("125.125.45.149"); list.Add("182.34.32.90"); list.Add("113.121.46.19"); list.Add("113.121.45.103"); list.Add("113.121.23.219"); list.Add("111.72.56.135"); list.Add("111.72.63.12"); list.Add("111.72.62.174"); list.Add("111.72.58.28"); list.Add("111.72.62.226"); list.Add("106.226.227.245"); list.Add("111.79.173.163"); list.Add("106.7.78.39"); list.Add("182.84.86.158"); list.Add("182.100.238.11"); list.Add("111.72.107.203"); List ss = new List(); for (int i = 0; i < 65535; i++) { ss.Add(i + 1); } int max1 = list.Count; int num1 = 0; list.ForEach(async p => { await Task.Run(() => { int max = 65535; int num = 0; //比赛 ss.ForEach(async p1 => { await Task.Run(() => { var html = new HttpHelper().GetHtml(new HttpItem { Url = url, Method = method, WebProxy = new WebProxy(p + ":" + p1) }); }); lock (g) { num++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num < max) { Monitor.Wait(g);//等待 } } }); lock (g) { num1++; Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。 } }); lock (g) { while (num1 < max1) { Monitor.Wait(g);//等待 } } //if (!isWebSoxket) //{ // var number = 0; // while (html.Html.IsEmpty() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1) // || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1) // || (!title.IsEmpty() && html.Html.IndexOf(title) == -1)) // { // number++; // if (number > 100) // return null; // //if (number > IpCount) // html = new HttpHelper().GetHtml(new HttpItem // { // Url = url, // Method = method, // WebProxy = new WebProxy(CommonHelper.GetIp()) // }); // } //} return ""; } #endregion #region 时间 /// /// 将c# DateTime时间格式转换为Unix时间戳格式 /// /// 时间 /// long public static long ConvertDateTimeToInt(System.DateTime time) { System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0)); long t = (time.Ticks - startTime.Ticks) / 10000; //除10000调整为13位 return t; } /// /// 时间戳转时间 /// /// /// public static DateTime ConvertIntToDateTime(string unixTimeStamp) { DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1)); long lTime = long.Parse(unixTimeStamp + "0000"); TimeSpan toNow = new TimeSpan(lTime); DateTime targetDt = dtStart.Add(toNow); return dtStart.Add(toNow); } #endregion /// /// 线程是否执行完成 /// /// public static bool ThreadsFinsh() { int maxWorkerThreads, workerThreads; int maxportThreads, portThreads; /* GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。 */ ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads); ThreadPool.GetAvailableThreads(out workerThreads, out portThreads); Thread.Sleep(3000); Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads)); if (maxWorkerThreads - workerThreads == 0) { Trace.WriteLine("加载完成!"); return true; } return false; } /// /// 线程是否执行完成 /// /// public static bool ThreadsFinsh_new() { int maxWorkerThreads, workerThreads; int maxportThreads, portThreads; /* GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。 而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。 通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。 */ ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads); ThreadPool.GetAvailableThreads(out workerThreads, out portThreads); Thread.Sleep(3000); Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads - 3)); if (maxWorkerThreads - workerThreads - 3 == 0) { Trace.WriteLine("加载完成!"); return true; } return false; } /// /// 文件写入 /// public static void Write(string path, string data) { using (var fs = new FileStream(path, FileMode.Append)) { using (var sw = new StreamWriter(fs)) { sw.WriteLine(data); sw.Flush(); } } } public static void Write_IP(string path, string data) { using (var fs = new FileStream(path, FileMode.Create)) { using (var sw = new StreamWriter(fs, Encoding.UTF8)) { sw.Write(data); sw.Flush(); } } } public static void LogBD(string content, string pathName = "",string directoryName="Log") { if (pathName.IsEmpty()) pathName = content; var path = AppDomain.CurrentDomain.BaseDirectory + "/"+ directoryName; CreateDirectory(path); path += $"/{DateTime.Now.ToString("yyyyMMdd")}"; CreateDirectory(path); Write(path + $"/{pathName}.txt", content + "||" + DateTime.Now.ToString()); } /// /// 创建文件夹 /// /// private static void CreateDirectory(string path) { if (!Directory.Exists(path)) Directory.CreateDirectory(path); } public static T Mapper(object data) { return AutoMapper.Mapper.DynamicMap(data); } } class AsyncSemaphore { private readonly static Task s_completed = Task.FromResult(true); private readonly Queue> m_waiters = new Queue>(); private int m_currentCount; public AsyncSemaphore(int initialCount) { if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount"); m_currentCount = initialCount; } public Task WaitAsync() { lock (m_waiters) { if (m_currentCount > 0) { --m_currentCount; return s_completed; } else { var waiter = new TaskCompletionSource(); m_waiters.Enqueue(waiter); return waiter.Task; } } } public void Release() { TaskCompletionSource toRelease = null; lock (m_waiters) { if (m_waiters.Count > 0) toRelease = m_waiters.Dequeue(); else ++m_currentCount; } if (toRelease != null) toRelease.SetResult(true); } } public class AsyncLock { private readonly AsyncSemaphore m_semaphore; private readonly Task m_releaser; public AsyncLock() { m_semaphore = new AsyncSemaphore(1); m_releaser = Task.FromResult(new Releaser(this)); } public Task LockAsync() { var wait = m_semaphore.WaitAsync(); return wait.IsCompleted ? m_releaser : wait.ContinueWith((_, state) => new Releaser((AsyncLock)state), this, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default); } public struct Releaser : IDisposable { private readonly AsyncLock m_toRelease; internal Releaser(AsyncLock toRelease) { m_toRelease = toRelease; } public void Dispose() { if (m_toRelease != null) m_toRelease.m_semaphore.Release(); } } } }