123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863 |
- using FCS.Models;
- using FCS.Models.DTO;
- using HtmlAgilityPack;
- using Newtonsoft.Json;
- using Quartz;
- using System;
- using System.Collections.Generic;
- using System.Configuration;
- using System.Diagnostics;
- using System.IO;
- using System.Linq;
- using System.Net;
- using System.Reflection;
- using System.Runtime.Remoting.Messaging;
- using System.Text;
- using System.Text.RegularExpressions;
- using System.Threading;
- using System.Threading.Tasks;
- using System.Xml;
- namespace FCS.Common
- {
-
-
-
- public static class CommonHelper
- {
-
-
-
-
-
-
-
- public static List<T> ConvertXMLToObject<T>(string FileName, string WrapperNodeName)
- {
- XmlDocument doc = new XmlDocument();
- doc.Load(FileName);
- List<T> result = new List<T>();
- var TType = typeof(T);
- XmlNodeList nodeList = doc.ChildNodes;
- if (!string.IsNullOrEmpty(WrapperNodeName))
- {
- foreach (XmlNode node in doc.ChildNodes)
- {
- if (node.Name == WrapperNodeName)
- {
- nodeList = node.ChildNodes;
- break;
- }
- }
- }
- object oneT = null;
- foreach (XmlNode node in nodeList)
- {
- if (node.NodeType == XmlNodeType.Comment || node.NodeType == XmlNodeType.XmlDeclaration) continue;
- oneT = TType.Assembly.CreateInstance(TType.FullName);
- foreach (XmlNode item in node.ChildNodes)
- {
- if (item.NodeType == XmlNodeType.Comment) continue;
- var property = TType.GetProperty(item.Name);
- if (property != null)
- property.SetValue(oneT, Convert.ChangeType(item.InnerText, property.PropertyType), null);
- }
- result.Add((T)oneT);
- }
- return result;
- }
-
-
-
-
-
- public static FCSConfig GetConfigFromDataMap(JobDataMap datamap)
- {
- FCSConfig config = new FCSConfig();
- var properties = typeof(FCSConfig).GetProperties();
- foreach (PropertyInfo info in properties)
- {
- if (info.PropertyType == typeof(string))
- info.SetValue(config, datamap.GetString(info.Name), null);
- else if (info.PropertyType == typeof(Int32))
- info.SetValue(config, datamap.GetInt(info.Name), null);
- }
- return config;
- }
- #region 日志信息
- public static string GetJobMainLogInfo(string QiHao)
- {
- return string.Format("通过主站地址抓取{0}期开奖数据成功", QiHao);
- }
- public static string GetJobLogError(string QiHao)
- {
- return string.Format("【{0}】抓取期开奖数据失败", QiHao);
- }
- #endregion 日志信息
-
-
-
-
-
-
- public static T ChangeType<T>(object value)
- {
- return ChangeType<T>(value, default(T));
- }
-
-
-
-
-
-
-
- public static T ChangeType<T>(object value, T defaultValue)
- {
- if (value != null)
- {
- Type nullableType = typeof(T);
- if (!nullableType.IsInterface && (!nullableType.IsClass || (nullableType == typeof(string))))
- {
- if (nullableType.IsGenericType && (nullableType.GetGenericTypeDefinition() == typeof(Nullable<>)))
- {
- return (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(nullableType));
- }
- if (nullableType.IsEnum)
- {
- return (T)Enum.Parse(nullableType, value.ToString());
- }
- return (T)Convert.ChangeType(value, nullableType);
- }
- if (value is T)
- {
- return (T)value;
- }
- }
- return defaultValue;
- }
-
-
-
-
-
-
- public static object ChangeType(object value, Type type)
- {
- if (value != null)
- {
- var nullableType = Nullable.GetUnderlyingType(type);
- if (nullableType != null)
- {
- return Convert.ChangeType(value, nullableType);
- }
- if (Convert.IsDBNull(value))
- return type.IsValueType ? Activator.CreateInstance(type) : null;
- return Convert.ChangeType(value, type);
- }
- return null;
- }
- #region 获取全局唯一GUID
-
-
-
-
-
-
-
-
-
-
- public static string GetGuid(bool needReplace = true, string format = "N")
- {
- Guid res = NewSequentialGuid();
- return needReplace ? res.ToString(format) : res.ToString();
- }
- [System.Runtime.InteropServices.DllImport("rpcrt4.dll", SetLastError = true)]
- static extern int UuidCreateSequential(byte[] buffer);
-
-
-
-
- private static Guid NewSequentialGuid()
- {
- byte[] raw = new byte[16];
- if (UuidCreateSequential(raw) != 0)
- throw new System.ComponentModel.Win32Exception(System.Runtime.InteropServices.Marshal.GetLastWin32Error());
- byte[] fix = new byte[16];
-
- fix[0x0] = raw[0x3];
- fix[0x1] = raw[0x2];
- fix[0x2] = raw[0x1];
- fix[0x3] = raw[0x0];
-
- fix[0x4] = raw[0x5];
- fix[0x5] = raw[0x4];
-
- fix[0x6] = raw[0x7];
- fix[0x7] = raw[0x6];
-
- fix[0x8] = raw[0x8];
- fix[0x9] = raw[0x9];
- fix[0xA] = raw[0xA];
- fix[0xB] = raw[0xB];
- fix[0xC] = raw[0xC];
- fix[0xD] = raw[0xD];
- fix[0xE] = raw[0xE];
- fix[0xF] = raw[0xF];
- return new Guid(fix);
- }
- #endregion 获取全局唯一GUID
- #region HtmlAgilityPack
- public static int IpCount;
- public static List<string> IpList;
- public static string GetIp(string path = "", bool isGetIp = true)
- {
- if (isGetIp && IpList.Count > 0)
- {
- var ran = new Random().Next(0, IpList.Count);
- return IpList[ran];
- }
- if (path.IsEmpty())
- path = AppDomain.CurrentDomain.BaseDirectory + "/XmlConfig/IP.txt";
- StreamReader sr;
- try
- {
- sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
- }
- catch (Exception)
- {
- Thread.Sleep(1000);
- sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
- }
- string content = sr.ReadToEnd().ToString();
- sr.Close();
- var list = content.JsonToList<string>();
- if (isGetIp)
- IpList = list;
- var ram = new Random().Next(0, list.Count);
- IpCount = list.Count;
- return list[ram];
- }
- static object locker = new object();
- static bool isGetIp = false;
- public delegate string GetIPDataBYOne(List<string> _urlList, string _title = "", bool isFormData = false);
- public delegate string GetIPDataBYOne_FormData(List<string> _urlList, Dictionary<string, string> formData, string _title = "");
-
-
-
-
-
- public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model)
- {
- var doc = new HtmlDocument();
- doc.LoadHtml(GetHtmlByIP(model));
- return doc;
- }
-
-
-
-
-
- public static string GetHtmlString(HtmlParameterDTO model)
- {
- return GetHtmlByIP(model);
- }
-
-
-
-
-
- private static string GetHtmlByIP(HtmlParameterDTO model)
- {
- var NotIpList = new List<string>();
- Stopwatch sw = new Stopwatch();
- sw.Start();
- var ip = model.IP.IsEmpty() ? GetIp() : model.IP;
- var httpItem = new HttpItem();
- lock (locker)
- {
- httpItem=Mapper<HttpItem>(model);
- httpItem.WebProxy = new WebProxy(ip);
- }
- var html = new HttpHelper().GetHtml(httpItem);
-
- while ((model.IsCheckEmpty && html.Html.IsEmpty())
- || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
- || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
- || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
- || html.Html.Contains("502 Bad Gateway")
- || html.Html.Contains("400 Bad Request")
- || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
- || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
- || html.Html.Contains("缓存访问被拒绝")
- || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title)))
- {
- if (html.Html.ToLower().Contains("exception report"))
- {
- return ConfigurationManager.AppSettings["Termination"].ToString();
- }
- NotIpList.Add(ip);
- if (NotIpList.Distinct().ToList().Count == IpCount || NotIpList.Distinct().ToList().Count > model.NotIpNumber)
- {
-
-
- LogBD(model.Url, "LogUrl", "UrlLog");
- return ConfigurationManager.AppSettings["Termination"].ToString();
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- }
- else
- {
- ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
- while (NotIpList.Contains(ip) && model.IP.IsEmpty())
- ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
- }
- httpItem.WebProxy = new WebProxy(ip);
- html = new HttpHelper().GetHtml(httpItem);
- }
- sw.Stop();
- Trace.WriteLine("url:" + model.Url + "||IP:" + ip + "||时间:" + sw.ElapsedMilliseconds + "毫秒");
- return html.Html;
- }
-
-
-
-
-
-
- public static HtmlDocument GetHtml(string url, string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get", int timeout = 90 * 1000, int notIpNUmber = 100)
- {
- return GetHtmlHtmlDocument(new HtmlParameterDTO
- {
- Url = url,
- Title = title,
- IP = webProxy,
- Method = method,
- Timeout = timeout,
- NotIpNumber = notIpNUmber
- });
- }
-
-
-
-
-
-
-
- public static HtmlDocument GetHtml(string url, Dictionary<string, string> formData, string title = "", string webProxy = "", int timeout = 90 * 1000, int notIpNUmber = 100)
- {
-
- return GetHtmlHtmlDocument(new HtmlParameterDTO
- {
- Url = url,
- Title = title,
- IP = webProxy,
- Method = "POST",
- Timeout = timeout,
- NotIpNumber = notIpNUmber,
- ContentType = "application/x-www-form-urlencoded",
- FormData = formData
- });
- }
- #endregion
- #region lg
-
-
-
-
-
-
- public static string GetHtmlString(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
- {
- var NotIpList = new List<string>();
- var ip = webProxy.IsEmpty() ? GetIp() : webProxy;
- var html = new HttpHelper().GetHtml(new HttpItem
- {
- Url = url,
- Method = method,
- WebProxy = new WebProxy(ip),
- Timeout = timeout
- });
- int number = 0;
- if (!isWebSoxket)
- {
- while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
- || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
- || (!title.IsEmpty() && html.Html.IndexOf(title) == -1)
- || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
- || (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
- || ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
- || html.Html.Contains("502 Bad Gateway")
- || html.Html.Contains("400 Bad Request")
- || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
- || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
- || html.Html.Contains("缓存访问被拒绝")
- || (title.IsEmpty() && !html.Html.Contains(title)))
- {
- number++;
- if (number > 40)
- return "";
- if (html.Html.ToLower().Contains("exception report"))
- {
- ConfigurationManager.AppSettings["Termination"].ToString();
- break;
- }
- NotIpList.Add(ip);
- if (NotIpList.Distinct().ToList().Count == IpCount)
- {
-
-
-
-
-
-
-
-
-
-
-
- return ConfigurationManager.AppSettings["Termination"].ToString();
-
-
-
-
- }
- else
- ip = webProxy.IsEmpty() ? GetIp() : webProxy;
- while (NotIpList.Contains(ip))
- ip = webProxy.IsEmpty() ? GetIp() : webProxy;
- html = new HttpHelper().GetHtml(new HttpItem
- {
- Url = url,
- Method = method,
- WebProxy = new WebProxy(ip)
- });
- }
- }
- return html.Html;
- }
-
-
-
-
-
-
- public static HttpResult GetPostHtmlString(string url, HttpItem model, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
- {
- model.Timeout = timeout;
- var html = new HttpHelper().GetHtml(model);
- if (!isWebSoxket)
- {
- var number = 0;
- while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
- || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
- || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
- {
- number++;
- if (number > 20)
- return html;
- model.WebProxy = new WebProxy(GetIp());
- html = new HttpHelper().GetHtml(model);
- }
- }
- return html;
- }
- public static string GetHtmlString_ceshi(string url = "http://fenxi.zgzcw.com/2321966/bjop", string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get")
- {
- var html3 = new HttpHelper().GetHtml(new HttpItem
- {
- Url = url,
- Method = method,
- WebProxy = new WebProxy("113.124.93.1:601")
- });
- html3 = new HttpHelper().GetHtml(new HttpItem
- {
- Url = url,
- Method = method,
- WebProxy = new WebProxy("117.91.249.95:601")
- });
- F_Grouping g = new F_Grouping();
- List<string> list = new List<string>();
- list.Add("27.26.162.129");
- list.Add("117.91.249.95");
- list.Add("222.189.190.117");
- list.Add("222.189.191.254");
- list.Add("111.72.57.234");
- list.Add("113.124.93.1");
- list.Add("60.189.167.102");
- list.Add("180.118.141.126");
- list.Add("221.230.123.45");
- list.Add("221.230.123.101");
- list.Add("221.230.124.127");
- list.Add("125.125.45.149");
- list.Add("182.34.32.90");
- list.Add("113.121.46.19");
- list.Add("113.121.45.103");
- list.Add("113.121.23.219");
- list.Add("111.72.56.135");
- list.Add("111.72.63.12");
- list.Add("111.72.62.174");
- list.Add("111.72.58.28");
- list.Add("111.72.62.226");
- list.Add("106.226.227.245");
- list.Add("111.79.173.163");
- list.Add("106.7.78.39");
- list.Add("182.84.86.158");
- list.Add("182.100.238.11");
- list.Add("111.72.107.203");
- List<int> ss = new List<int>();
- for (int i = 0; i < 65535; i++)
- {
- ss.Add(i + 1);
- }
- int max1 = list.Count;
- int num1 = 0;
- list.ForEach(async p =>
- {
- await Task.Run(() =>
- {
- int max = 65535;
- int num = 0;
-
- ss.ForEach(async p1 =>
- {
- await Task.Run(() =>
- {
- var html = new HttpHelper().GetHtml(new HttpItem
- {
- Url = url,
- Method = method,
- WebProxy = new WebProxy(p + ":" + p1)
- });
- });
- lock (g)
- {
- num++;
- Monitor.Pulse(g);
- }
- });
- lock (g)
- {
- while (num < max)
- {
- Monitor.Wait(g);
- }
- }
- });
- lock (g)
- {
- num1++;
- Monitor.Pulse(g);
- }
- });
- lock (g)
- {
- while (num1 < max1)
- {
- Monitor.Wait(g);
- }
- }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- return "";
- }
- #endregion
- #region 时间
-
-
-
-
-
- public static long ConvertDateTimeToInt(System.DateTime time)
- {
- System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0));
- long t = (time.Ticks - startTime.Ticks) / 10000;
- return t;
- }
-
-
-
-
-
- public static DateTime ConvertIntToDateTime(string unixTimeStamp)
- {
- DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
- long lTime = long.Parse(unixTimeStamp + "0000");
- TimeSpan toNow = new TimeSpan(lTime);
- DateTime targetDt = dtStart.Add(toNow);
- return dtStart.Add(toNow);
- }
- #endregion
-
-
-
-
- public static bool ThreadsFinsh()
- {
- int maxWorkerThreads, workerThreads;
- int maxportThreads, portThreads;
-
- ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
- ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
- Thread.Sleep(3000);
- Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
- if (maxWorkerThreads - workerThreads == 0)
- {
- Trace.WriteLine("加载完成!");
- return true;
- }
- return false;
- }
-
-
-
-
- public static bool ThreadsFinsh_new()
- {
- int maxWorkerThreads, workerThreads;
- int maxportThreads, portThreads;
-
- ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
- ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
- Thread.Sleep(3000);
- Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads - 3));
- if (maxWorkerThreads - workerThreads - 3 == 0)
- {
- Trace.WriteLine("加载完成!");
- return true;
- }
- return false;
- }
-
-
-
- public static void Write(string path, string data)
- {
- using (var fs = new FileStream(path, FileMode.Append))
- {
- using (var sw = new StreamWriter(fs))
- {
- sw.WriteLine(data);
- sw.Flush();
- }
- }
- }
- public static void Write_IP(string path, string data)
- {
- using (var fs = new FileStream(path, FileMode.Create))
- {
- using (var sw = new StreamWriter(fs, Encoding.UTF8))
- {
- sw.Write(data);
- sw.Flush();
- }
- }
- }
- public static void LogBD(string content, string pathName = "",string directoryName="Log")
- {
- if (pathName.IsEmpty())
- pathName = content;
- var path = AppDomain.CurrentDomain.BaseDirectory + "/"+ directoryName;
- CreateDirectory(path);
- path += $"/{DateTime.Now.ToString("yyyyMMdd")}";
- CreateDirectory(path);
- Write(path + $"/{pathName}.txt", content + "||" + DateTime.Now.ToString());
- }
-
-
-
-
- private static void CreateDirectory(string path)
- {
- if (!Directory.Exists(path))
- Directory.CreateDirectory(path);
- }
- public static T Mapper<T>(object data)
- {
- return AutoMapper.Mapper.DynamicMap<T>(data);
- }
- }
- class AsyncSemaphore
- {
- private readonly static Task s_completed = Task.FromResult(true);
- private readonly Queue<TaskCompletionSource<bool>> m_waiters = new Queue<TaskCompletionSource<bool>>();
- private int m_currentCount;
- public AsyncSemaphore(int initialCount)
- {
- if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount");
- m_currentCount = initialCount;
- }
- public Task WaitAsync()
- {
- lock (m_waiters)
- {
- if (m_currentCount > 0)
- {
- --m_currentCount;
- return s_completed;
- }
- else
- {
- var waiter = new TaskCompletionSource<bool>();
- m_waiters.Enqueue(waiter);
- return waiter.Task;
- }
- }
- }
- public void Release()
- {
- TaskCompletionSource<bool> toRelease = null;
- lock (m_waiters)
- {
- if (m_waiters.Count > 0)
- toRelease = m_waiters.Dequeue();
- else
- ++m_currentCount;
- }
- if (toRelease != null)
- toRelease.SetResult(true);
- }
- }
- public class AsyncLock
- {
- private readonly AsyncSemaphore m_semaphore;
- private readonly Task<Releaser> m_releaser;
- public AsyncLock()
- {
- m_semaphore = new AsyncSemaphore(1);
- m_releaser = Task.FromResult(new Releaser(this));
- }
- public Task<Releaser> LockAsync()
- {
- var wait = m_semaphore.WaitAsync();
- return wait.IsCompleted ?
- m_releaser :
- wait.ContinueWith((_, state) => new Releaser((AsyncLock)state),
- this, CancellationToken.None,
- TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
- }
- public struct Releaser : IDisposable
- {
- private readonly AsyncLock m_toRelease;
- internal Releaser(AsyncLock toRelease) { m_toRelease = toRelease; }
- public void Dispose()
- {
- if (m_toRelease != null)
- m_toRelease.m_semaphore.Release();
- }
- }
- }
- }
|