using FCS.Models;
using FCS.Models.DTO;
using HtmlAgilityPack;
using Newtonsoft.Json;
using Quartz;
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Runtime.Remoting.Messaging;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
namespace FCS.Common
{
///
/// 公用帮助类
///
public static class CommonHelper
{
///
/// 将XML内容转换成目标对象实体集合
///
/// 目标对象实体
/// 完整文件名(根目录下只需文件名称)
///
///
public static List ConvertXMLToObject(string FileName, string WrapperNodeName)
{
XmlDocument doc = new XmlDocument();
doc.Load(FileName);
List result = new List();
var TType = typeof(T);
XmlNodeList nodeList = doc.ChildNodes;
if (!string.IsNullOrEmpty(WrapperNodeName))
{
foreach (XmlNode node in doc.ChildNodes)
{
if (node.Name == WrapperNodeName)
{
nodeList = node.ChildNodes;
break;
}
}
}
object oneT = null;
foreach (XmlNode node in nodeList)
{
if (node.NodeType == XmlNodeType.Comment || node.NodeType == XmlNodeType.XmlDeclaration) continue;
oneT = TType.Assembly.CreateInstance(TType.FullName);
foreach (XmlNode item in node.ChildNodes)
{
if (item.NodeType == XmlNodeType.Comment) continue;
var property = TType.GetProperty(item.Name);
if (property != null)
property.SetValue(oneT, Convert.ChangeType(item.InnerText, property.PropertyType), null);
}
result.Add((T)oneT);
}
return result;
}
///
/// 从作业数据地图中获取配置信息
///
/// 作业数据地图
///
public static FCSConfig GetConfigFromDataMap(JobDataMap datamap)
{
FCSConfig config = new FCSConfig();
var properties = typeof(FCSConfig).GetProperties();
foreach (PropertyInfo info in properties)
{
if (info.PropertyType == typeof(string))
info.SetValue(config, datamap.GetString(info.Name), null);
else if (info.PropertyType == typeof(Int32))
info.SetValue(config, datamap.GetInt(info.Name), null);
}
return config;
}
#region 日志信息
public static string GetJobMainLogInfo(string QiHao)
{
return string.Format("通过主站地址抓取{0}期开奖数据成功", QiHao);
}
public static string GetJobLogError(string QiHao)
{
return string.Format("【{0}】抓取期开奖数据失败", QiHao);
}
#endregion 日志信息
///
/// 将值转换为T类型数据
///
/// 目标类型
/// 数据值
///
public static T ChangeType(object value)
{
return ChangeType(value, default(T));
}
///
/// 将值转换为T类型数据,失败则返回T类型默认值
///
/// 目标类型
/// 数据值
/// T类型默认值
///
public static T ChangeType(object value, T defaultValue)
{
if (value != null)
{
Type nullableType = typeof(T);
if (!nullableType.IsInterface && (!nullableType.IsClass || (nullableType == typeof(string))))
{
if (nullableType.IsGenericType && (nullableType.GetGenericTypeDefinition() == typeof(Nullable<>)))
{
return (T)Convert.ChangeType(value, Nullable.GetUnderlyingType(nullableType));
}
if (nullableType.IsEnum)
{
return (T)Enum.Parse(nullableType, value.ToString());
}
return (T)Convert.ChangeType(value, nullableType);
}
if (value is T)
{
return (T)value;
}
}
return defaultValue;
}
///
/// 将值转换为type类型的值
///
/// 值
/// 目标类型
///
public static object ChangeType(object value, Type type)
{
if (value != null)
{
var nullableType = Nullable.GetUnderlyingType(type);
if (nullableType != null)//可空
{
return Convert.ChangeType(value, nullableType);
}
if (Convert.IsDBNull(value))//特殊处理,由于数据库类型与项目中的类型定义不匹配
return type.IsValueType ? Activator.CreateInstance(type) : null;
return Convert.ChangeType(value, type);
}
return null;
}
#region 获取全局唯一GUID
///
/// 获取全局唯一GUID
///
/// 是否需要替换-
/// 格式化
/// N:38bddf48f43c48588e0d78761eaa1ce6>
/// P:(778406c2-efff-4262-ab03-70a77d09c2b5)>
/// B:{09f140d5-af72-44ba-a763-c861304b46f8}>
/// D:57d99d89-caab-482a-a0e9-a0a803eed3ba>
///
public static string GetGuid(bool needReplace = true, string format = "N")
{
Guid res = NewSequentialGuid();//Guid.NewGuid();
return needReplace ? res.ToString(format) : res.ToString();
}
[System.Runtime.InteropServices.DllImport("rpcrt4.dll", SetLastError = true)]
static extern int UuidCreateSequential(byte[] buffer);
///
/// 创建有序GUID
///
///
private static Guid NewSequentialGuid()
{
byte[] raw = new byte[16];
if (UuidCreateSequential(raw) != 0)
throw new System.ComponentModel.Win32Exception(System.Runtime.InteropServices.Marshal.GetLastWin32Error());
byte[] fix = new byte[16];
// reverse 0..3
fix[0x0] = raw[0x3];
fix[0x1] = raw[0x2];
fix[0x2] = raw[0x1];
fix[0x3] = raw[0x0];
// reverse 4 & 5
fix[0x4] = raw[0x5];
fix[0x5] = raw[0x4];
// reverse 6 & 7
fix[0x6] = raw[0x7];
fix[0x7] = raw[0x6];
// all other are unchanged
fix[0x8] = raw[0x8];
fix[0x9] = raw[0x9];
fix[0xA] = raw[0xA];
fix[0xB] = raw[0xB];
fix[0xC] = raw[0xC];
fix[0xD] = raw[0xD];
fix[0xE] = raw[0xE];
fix[0xF] = raw[0xF];
return new Guid(fix);
}
#endregion 获取全局唯一GUID
#region HtmlAgilityPack
public static int IpCount;
public static List IpList;
public static string GetIp(string path = "", bool isGetIp = true)
{
if (isGetIp && IpList.Count > 0)
{
var ran = new Random().Next(0, IpList.Count);
return IpList[ran];
}
if (path.IsEmpty())
path = AppDomain.CurrentDomain.BaseDirectory + "/XmlConfig/IP.txt";
StreamReader sr;
try
{
sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
}
catch (Exception)
{
Thread.Sleep(1000);
sr = new StreamReader(path, System.Text.Encoding.GetEncoding("utf-8"));
}
string content = sr.ReadToEnd().ToString();
sr.Close();
var list = content.JsonToList();
if (isGetIp)
IpList = list;
var ram = new Random().Next(0, list.Count);
IpCount = list.Count;
return list[ram];
}
static object locker = new object();
static bool isGetIp = false;
public delegate string GetIPDataBYOne(List _urlList, string _title = "", bool isFormData = false);
public delegate string GetIPDataBYOne_FormData(List _urlList, Dictionary formData, string _title = "");
///
/// 获取HTML
///
/// 参数实体
///
public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model)
{
var doc = new HtmlDocument();
doc.LoadHtml(GetHtmlByIP(model));
return doc;
}
///
/// 获取HTML
///
/// 参数实体
///
public static string GetHtmlString(HtmlParameterDTO model)
{
return GetHtmlByIP(model);
}
///
/// 通过Ip获取页面的HTML
///
///
///
private static string GetHtmlByIP(HtmlParameterDTO model)
{
var NotIpList = new List();
Stopwatch sw = new Stopwatch();
sw.Start();
var ip = model.IP.IsEmpty() ? GetIp() : model.IP;
var httpItem = new HttpItem();
lock (locker)
{
httpItem=Mapper(model);
httpItem.WebProxy = new WebProxy(ip);
}
var html = new HttpHelper().GetHtml(httpItem);
//对文本的检查
while ((model.IsCheckEmpty && html.Html.IsEmpty())
|| html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
|| (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
|| ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
|| html.Html.Contains("502 Bad Gateway")
|| html.Html.Contains("400 Bad Request")
|| (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
|| (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
|| html.Html.Contains("缓存访问被拒绝")
|| (!model.Title.IsEmpty() && !html.Html.Contains(model.Title)))
{
if (html.Html.ToLower().Contains("exception report"))
{
return ConfigurationManager.AppSettings["Termination"].ToString();
}
NotIpList.Add(ip);
if (NotIpList.Distinct().ToList().Count == IpCount || NotIpList.Distinct().ToList().Count > model.NotIpNumber)
{
//EmailHelper.Send("1625453870@qq.com", "IP用完,未获取值的URL", "URL:" + model.Url + "||参数:" + model.FormData.TryToJson());
//return ConfigurationManager.AppSettings["Termination"].ToString();
LogBD(model.Url, "LogUrl", "UrlLog");
return ConfigurationManager.AppSettings["Termination"].ToString();
//IAsyncResult asyncResult;
//lock (locker)
//{
// GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
// asyncResult = task.BeginInvoke(new List { model.Url }, model.Title, false, null, null);
// while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
// {
// }
// ip = task.EndInvoke(asyncResult);
// if (ip.IsEmpty())
// return ConfigurationManager.AppSettings["Termination"].ToString();
// else
// return ip;
//}
}
else
{
ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
while (NotIpList.Contains(ip) && model.IP.IsEmpty())
ip = ip = model.IP.IsEmpty() ? GetIp() : model.IP;
}
httpItem.WebProxy = new WebProxy(ip);
html = new HttpHelper().GetHtml(httpItem);
}
sw.Stop();
Trace.WriteLine("url:" + model.Url + "||IP:" + ip + "||时间:" + sw.ElapsedMilliseconds + "毫秒");
return html.Html;
}
///
/// 得到HtmlDocument
///
///
///
///
public static HtmlDocument GetHtml(string url, string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get", int timeout = 90 * 1000, int notIpNUmber = 100)
{
return GetHtmlHtmlDocument(new HtmlParameterDTO
{
Url = url,
Title = title,
IP = webProxy,
Method = method,
Timeout = timeout,
NotIpNumber = notIpNUmber
});
}
///
/// 得到HtmlDocument
/// From表单提交
///
///
///
///
public static HtmlDocument GetHtml(string url, Dictionary formData, string title = "", string webProxy = "", int timeout = 90 * 1000, int notIpNUmber = 100)
{
// ContentType = "application/x-www-form-urlencoded",
return GetHtmlHtmlDocument(new HtmlParameterDTO
{
Url = url,
Title = title,
IP = webProxy,
Method = "POST",
Timeout = timeout,
NotIpNumber = notIpNUmber,
ContentType = "application/x-www-form-urlencoded",
FormData = formData
});
}
#endregion
#region lg
///
/// 得到HtmlDocument
///
///
///
///
public static string GetHtmlString(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
{
var NotIpList = new List();
var ip = webProxy.IsEmpty() ? GetIp() : webProxy;
var html = new HttpHelper().GetHtml(new HttpItem
{
Url = url,
Method = method,
WebProxy = new WebProxy(ip),
Timeout = timeout
});
int number = 0;
if (!isWebSoxket)
{
while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
|| (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
|| (!title.IsEmpty() && html.Html.IndexOf(title) == -1)
|| html.Html == ConfigurationManager.AppSettings["HttpException"].ToString()
|| (html.Html.Contains("403") && html.Html.ToLower().Contains("forbidden"))
|| ((html.Html.Contains("HTTP Status 404") || html.Html.Contains("404 Not Found")) && html.Html.ToLower().Contains("not found"))
|| html.Html.Contains("502 Bad Gateway")
|| html.Html.Contains("400 Bad Request")
|| (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
|| (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
|| html.Html.Contains("缓存访问被拒绝")
|| (title.IsEmpty() && !html.Html.Contains(title)))
{
number++;
if (number > 40)
return "";
if (html.Html.ToLower().Contains("exception report"))
{
ConfigurationManager.AppSettings["Termination"].ToString();
break;
}
NotIpList.Add(ip);
if (NotIpList.Distinct().ToList().Count == IpCount)
{
//IAsyncResult asyncResult;
//lock (locker)
//{
// GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
// asyncResult = task.BeginInvoke(new List { url }, title, false, null, null);
// while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
// {
// }
// ip = task.EndInvoke(asyncResult);
// if (ip.IsEmpty())
// {
return ConfigurationManager.AppSettings["Termination"].ToString();
// break;
// }
//}
//}
}
else
ip = webProxy.IsEmpty() ? GetIp() : webProxy;
while (NotIpList.Contains(ip))
ip = webProxy.IsEmpty() ? GetIp() : webProxy;
html = new HttpHelper().GetHtml(new HttpItem
{
Url = url,
Method = method,
WebProxy = new WebProxy(ip)
});
}
}
return html.Html;
}
///
/// 得到HtmlDocument
///
///
///
///
public static HttpResult GetPostHtmlString(string url, HttpItem model, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
{
model.Timeout = timeout;
var html = new HttpHelper().GetHtml(model);
if (!isWebSoxket)
{
var number = 0;
while (html.Html.IsEmpty() || html.Html == ConfigurationManager.AppSettings["HttpException"].ToString() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
|| (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
|| (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
{
number++;
if (number > 20)
return html;
model.WebProxy = new WebProxy(GetIp());
html = new HttpHelper().GetHtml(model);
}
}
return html;
}
public static string GetHtmlString_ceshi(string url = "http://fenxi.zgzcw.com/2321966/bjop", string title = "", bool isWebSoxket = false, string webProxy = "", string method = "get")
{
var html3 = new HttpHelper().GetHtml(new HttpItem
{
Url = url,
Method = method,
WebProxy = new WebProxy("113.124.93.1:601")
});
html3 = new HttpHelper().GetHtml(new HttpItem
{
Url = url,
Method = method,
WebProxy = new WebProxy("117.91.249.95:601")
});
F_Grouping g = new F_Grouping();
List list = new List();
list.Add("27.26.162.129");
list.Add("117.91.249.95");
list.Add("222.189.190.117");
list.Add("222.189.191.254");
list.Add("111.72.57.234");
list.Add("113.124.93.1");
list.Add("60.189.167.102");
list.Add("180.118.141.126");
list.Add("221.230.123.45");
list.Add("221.230.123.101");
list.Add("221.230.124.127");
list.Add("125.125.45.149");
list.Add("182.34.32.90");
list.Add("113.121.46.19");
list.Add("113.121.45.103");
list.Add("113.121.23.219");
list.Add("111.72.56.135");
list.Add("111.72.63.12");
list.Add("111.72.62.174");
list.Add("111.72.58.28");
list.Add("111.72.62.226");
list.Add("106.226.227.245");
list.Add("111.79.173.163");
list.Add("106.7.78.39");
list.Add("182.84.86.158");
list.Add("182.100.238.11");
list.Add("111.72.107.203");
List ss = new List();
for (int i = 0; i < 65535; i++)
{
ss.Add(i + 1);
}
int max1 = list.Count;
int num1 = 0;
list.ForEach(async p =>
{
await Task.Run(() =>
{
int max = 65535;
int num = 0;
//比赛
ss.ForEach(async p1 =>
{
await Task.Run(() =>
{
var html = new HttpHelper().GetHtml(new HttpItem
{
Url = url,
Method = method,
WebProxy = new WebProxy(p + ":" + p1)
});
});
lock (g)
{
num++;
Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
}
});
lock (g)
{
while (num < max)
{
Monitor.Wait(g);//等待
}
}
});
lock (g)
{
num1++;
Monitor.Pulse(g); //完成,通知等待队列,告知已完,执行下一个。
}
});
lock (g)
{
while (num1 < max1)
{
Monitor.Wait(g);//等待
}
}
//if (!isWebSoxket)
//{
// var number = 0;
// while (html.Html.IsEmpty() || (html.Html.IndexOf("403") != -1 && html.Html.ToLower().IndexOf("forbidden") != -1)
// || (html.Html.IndexOf("HTTP Status 404") != -1 && html.Html.ToLower().IndexOf("not found") != -1)
// || (!title.IsEmpty() && html.Html.IndexOf(title) == -1))
// {
// number++;
// if (number > 100)
// return null;
// //if (number > IpCount)
// html = new HttpHelper().GetHtml(new HttpItem
// {
// Url = url,
// Method = method,
// WebProxy = new WebProxy(CommonHelper.GetIp())
// });
// }
//}
return "";
}
#endregion
#region 时间
///
/// 将c# DateTime时间格式转换为Unix时间戳格式
///
/// 时间
/// long
public static long ConvertDateTimeToInt(System.DateTime time)
{
System.DateTime startTime = TimeZone.CurrentTimeZone.ToLocalTime(new System.DateTime(1970, 1, 1, 0, 0, 0, 0));
long t = (time.Ticks - startTime.Ticks) / 10000; //除10000调整为13位
return t;
}
///
/// 时间戳转时间
///
///
///
public static DateTime ConvertIntToDateTime(string unixTimeStamp)
{
DateTime dtStart = TimeZone.CurrentTimeZone.ToLocalTime(new DateTime(1970, 1, 1));
long lTime = long.Parse(unixTimeStamp + "0000");
TimeSpan toNow = new TimeSpan(lTime);
DateTime targetDt = dtStart.Add(toNow);
return dtStart.Add(toNow);
}
#endregion
///
/// 线程是否执行完成
///
///
public static bool ThreadsFinsh()
{
int maxWorkerThreads, workerThreads;
int maxportThreads, portThreads;
/*
GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
*/
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
Thread.Sleep(3000);
Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
if (maxWorkerThreads - workerThreads == 0)
{
Trace.WriteLine("加载完成!");
return true;
}
return false;
}
///
/// 线程是否执行完成
///
///
public static bool ThreadsFinsh_new()
{
int maxWorkerThreads, workerThreads;
int maxportThreads, portThreads;
/*
GetAvailableThreads():检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
通过最大数目减可用数目就可以得到当前活动线程的数目,如果为零,那就说明没有活动线程,说明所有线程运行完毕。
*/
ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
Thread.Sleep(3000);
Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads - 3));
if (maxWorkerThreads - workerThreads - 3 == 0)
{
Trace.WriteLine("加载完成!");
return true;
}
return false;
}
///
/// 文件写入
///
public static void Write(string path, string data)
{
using (var fs = new FileStream(path, FileMode.Append))
{
using (var sw = new StreamWriter(fs))
{
sw.WriteLine(data);
sw.Flush();
}
}
}
public static void Write_IP(string path, string data)
{
using (var fs = new FileStream(path, FileMode.Create))
{
using (var sw = new StreamWriter(fs, Encoding.UTF8))
{
sw.Write(data);
sw.Flush();
}
}
}
public static void LogBD(string content, string pathName = "",string directoryName="Log")
{
if (pathName.IsEmpty())
pathName = content;
var path = AppDomain.CurrentDomain.BaseDirectory + "/"+ directoryName;
CreateDirectory(path);
path += $"/{DateTime.Now.ToString("yyyyMMdd")}";
CreateDirectory(path);
Write(path + $"/{pathName}.txt", content + "||" + DateTime.Now.ToString());
}
///
/// 创建文件夹
///
///
private static void CreateDirectory(string path)
{
if (!Directory.Exists(path))
Directory.CreateDirectory(path);
}
public static T Mapper(object data)
{
return AutoMapper.Mapper.DynamicMap(data);
}
}
class AsyncSemaphore
{
private readonly static Task s_completed = Task.FromResult(true);
private readonly Queue> m_waiters = new Queue>();
private int m_currentCount;
public AsyncSemaphore(int initialCount)
{
if (initialCount < 0) throw new ArgumentOutOfRangeException("initialCount");
m_currentCount = initialCount;
}
public Task WaitAsync()
{
lock (m_waiters)
{
if (m_currentCount > 0)
{
--m_currentCount;
return s_completed;
}
else
{
var waiter = new TaskCompletionSource();
m_waiters.Enqueue(waiter);
return waiter.Task;
}
}
}
public void Release()
{
TaskCompletionSource toRelease = null;
lock (m_waiters)
{
if (m_waiters.Count > 0)
toRelease = m_waiters.Dequeue();
else
++m_currentCount;
}
if (toRelease != null)
toRelease.SetResult(true);
}
}
public class AsyncLock
{
private readonly AsyncSemaphore m_semaphore;
private readonly Task m_releaser;
public AsyncLock()
{
m_semaphore = new AsyncSemaphore(1);
m_releaser = Task.FromResult(new Releaser(this));
}
public Task LockAsync()
{
var wait = m_semaphore.WaitAsync();
return wait.IsCompleted ?
m_releaser :
wait.ContinueWith((_, state) => new Releaser((AsyncLock)state),
this, CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default);
}
public struct Releaser : IDisposable
{
private readonly AsyncLock m_toRelease;
internal Releaser(AsyncLock toRelease) { m_toRelease = toRelease; }
public void Dispose()
{
if (m_toRelease != null)
m_toRelease.m_semaphore.Release();
}
}
}
}