1

FCS.Common/Helper/CommonHelper.cs

@@ -275,8 +275,13 @@ namespace FCS.Common
         /// <returns></returns>
         public static HtmlDocument GetHtmlHtmlDocument(HtmlParameterDTO model)
         {
+            var html = string.Empty;
+            if (model.IsWebClient)
+                html = GetHtmlByWebClient(model);
+            else
+                html = GetHtmlByIP(model);
             var doc = new HtmlDocument();
-            doc.LoadHtml(GetHtmlByIP(model));
+            doc.LoadHtml(html);
             return doc;
         }
         /// <summary>
@@ -286,7 +291,10 @@ namespace FCS.Common
         /// <returns></returns>
         public static string GetHtmlString(HtmlParameterDTO model)
         {
-            return GetHtmlByIP(model);
+            if (model.IsWebClient)
+                return GetHtmlByWebClient(model);
+            else
+                return GetHtmlByIP(model);
         }
 
         /// <summary>
@@ -304,7 +312,7 @@ namespace FCS.Common
             var httpItem = new HttpItem();
             lock (locker)
             {
-                httpItem=Mapper<HttpItem>(model);
+                httpItem = Mapper<HttpItem>(model);
                 httpItem.WebProxy = new WebProxy(ip);
             }
             var html = new HttpHelper().GetHtml(httpItem);
@@ -319,7 +327,7 @@ namespace FCS.Common
                 || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
                 || html.Html.Contains("缓存访问被拒绝")
                 || html.Html.Contains("无效用户")
-                || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title))) 
+                || (!model.Title.IsEmpty() && !html.Html.Contains(model.Title)))
             {
                 if (html.Html.ToLower().Contains("exception report"))
                 {
@@ -406,6 +414,20 @@ namespace FCS.Common
         }
         #endregion
 
+        #region WebClient请求
+        private static string GetHtmlByWebClient(HtmlParameterDTO model)
+        {
+            using (var webClient = new WebClient())
+            {
+                webClient.Credentials = CredentialCache.DefaultCredentials;
+                byte[] pageDate = webClient.DownloadData(model.Url);
+                var content = Encoding.UTF8.GetString(pageDate);
+                webClient.Dispose();
+                return content;
+            }
+        }
+        #endregion
+
         #region lg
 
         /// <summary>
@@ -578,23 +600,23 @@ namespace FCS.Common
                     int num = 0;
                     //比赛
                     ss.ForEach(async p1 =>
-                    {
-                        await Task.Run(() =>
-                        {
-                            var html = new HttpHelper().GetHtml(new HttpItem
-                            {
-                                Url = url,
-                                Method = method,
-                                WebProxy = new WebProxy(p + ":" + p1)
-                            });
-
-                        });
-                        lock (g)
-                        {
-                            num++;
-                            Monitor.Pulse(g); //完成，通知等待队列,告知已完，执行下一个。
-                        }
-                    });
+            {
+                await Task.Run(() =>
+        {
+            var html = new HttpHelper().GetHtml(new HttpItem
+            {
+                Url = url,
+                Method = method,
+                WebProxy = new WebProxy(p + ":" + p1)
+            });
+
+        });
+                lock (g)
+                {
+                    num++;
+                    Monitor.Pulse(g); //完成，通知等待队列,告知已完，执行下一个。
+                }
+            });
                     lock (g)
                     {
                         while (num < max)
@@ -753,11 +775,11 @@ namespace FCS.Common
             }
         }
 
-        public static void LogBD(string content, string pathName = "",string directoryName="Log")
+        public static void LogBD(string content, string pathName = "", string directoryName = "Log")
         {
             if (pathName.IsEmpty())
                 pathName = content;
-            var path = AppDomain.CurrentDomain.BaseDirectory + "/"+ directoryName;
+            var path = AppDomain.CurrentDomain.BaseDirectory + "/" + directoryName;
             CreateDirectory(path);
             path += $"/{DateTime.Now.ToString("yyyyMMdd")}";
             CreateDirectory(path);
@@ -778,6 +800,25 @@ namespace FCS.Common
         {
             return AutoMapper.Mapper.DynamicMap<T>(data);
         }
+
+        //获取外网IP
+        public static string GetExternalIP()
+        {
+            using (var webClient = new WebClient())
+            {
+                try
+                {
+                    webClient.Credentials = CredentialCache.DefaultCredentials;
+                    byte[] pageDate = webClient.DownloadData("http://pv.sohu.com/cityjson?ie=utf-8");
+                    String ip = Encoding.UTF8.GetString(pageDate); webClient.Dispose();
+                    Match rebool = Regex.Match(ip, @"\d{2,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}"); return rebool.Value;
+                }
+                catch (Exception e)
+                {
+                    return "";
+                }
+            }
+        }
     }
 
     class AsyncSemaphore

FCS.Crawler/Content/Jobs/BasketJobs/Basketball/B_GroupingJob.cs

@@ -11,6 +11,8 @@ using FCS.Models.Entity;
 using HtmlAgilityPack;
 using System.Threading;
 using Newtonsoft.Json;
+using System.Net;
+using System.Text.RegularExpressions;
 
 namespace FCS.Crawler.Basketball
 {
@@ -18,7 +20,7 @@ namespace FCS.Crawler.Basketball
     {
         private string eventId = "";
 
-   
+
         public void Click()
         {
             Click_NBA();
@@ -154,12 +156,43 @@ namespace FCS.Crawler.Basketball
         #endregion
 
         #region CBA
-        public void Click_CBA() {
+        public void Click_CBA()
+        {
+            //http://nba.nowscore.com/jsData/matchResult/17-18/l5_3.js?version=2018111310 季前赛
+            //http://nba.nowscore.com/jsData/matchResult/17-18/l5_1_2018_2.js?version=2018111310  常规赛   
+            //
+            //http://nba.nowscore.com/jsData/matchResult/17-18/l5_2.js?version=2018111310 季后赛
             var result = new List<B_Grouping>();
             eventId = new B_EventsJob().GetEventList().Where(p => p.Name == ConfigurationManager.AppSettings["CBAEventName"]).ToList()[0].Id;
-            
+            var seasonList = GetCBASeason().OrderByDescending(p=>p.Season).ToList();
+            seasonList.ForEach(p =>
+            {
+                //taskList.Add(Task.Run(() =>
+                //{
+                var url = $"http://nba.nowscore.com/cn/Playoffs.aspx?SclassID=5&matchSeason={p.Season}";
+                if(seasonList.IndexOf(p)==0)
+                    url = $"http://nba.nowscore.com/cn/Normal.aspx?SclassID=5&matchSeason={p.Season}";
+                var doc = CommonHelper.GetHtmlHtmlDocument(new Models.DTO.HtmlParameterDTO { Url = url, IsWebClient = true });
+                var scripts = doc.DocumentNode.SelectNodes(".//script");
+                foreach (var item in scripts)
+                {
+                    if (!item.Attributes["src"].IsEmpty() && item.Attributes["src"].Value.Contains("jsData") && item.Attributes["src"].Value.Contains(p.Season.Replace("20", "")))
+                    {
+                        url = item.Attributes["src"].Value;
+                        break;
+                    }
+
+                }
+                url = "http://nba.nowscore.com" + url;
+                var html = CommonHelper.GetHtmlString(new Models.DTO.HtmlParameterDTO { Url = url, IsWebClient = true });
+                var aa = 1;
+
+                // }));
+            });
         }
 
+
+
         /// <summary>
         /// 得到NBA的期数
         /// </summary>
@@ -171,15 +204,20 @@ namespace FCS.Crawler.Basketball
                              group a by a.Season into g
                              select g.Key).ToList();
             var url = "http://nba.nowscore.com/jsData/LeagueSeason/sea5.js";
-            var html = CommonHelper.GetHtmlString(new Models.DTO.HtmlParameterDTO { Url = url });
-            var data = JsonConvert.DeserializeObject<string[][]>(html);
+
+
+
+            var html = CommonHelper.GetHtmlString(new Models.DTO.HtmlParameterDTO { Url = url, IsWebClient = true });
+            var data = JsonConvert.DeserializeObject<string[][]>(html.Split('=')[1].Replace(";", ""));
             foreach (var item in data)
             {
-                yield return new B_Grouping
-                {
-                    Season = item[0].ToString(),
-                    Remark = item[1].ToString(),
-                };
+
+                if (!oldSeason.Contains(item[0]) || item[0].Contains(DateTime.Now.Year.ToString()))
+                    yield return new B_Grouping
+                    {
+                        Season = item[0].ToString(),
+                        Remark = item[1].ToString(),
+                    };
             }
         }
         #endregion

FCS.Crawler/Program.cs

@@ -34,7 +34,7 @@ namespace FCS.Crawler
                         new Service1()
              };
             ServiceBase.Run(ServicesToRun);
-            //new AgainstJob().Click();
+            //new B_GroupingJob().Click_CBA();
         }
 
 

FCS.Models/DTO/HtmlParameterDTO.cs

@@ -47,6 +47,9 @@ namespace FCS.Models.DTO
         /// 最多异常多少IP，就会进入获取在线IP得到html
         /// </summary>
         public int NotIpNumber { get; set; } = 500;
-      
+        /// <summary>
+        /// WebClient请求（默认false：否，true：是）
+        /// </summary>
+        public bool IsWebClient { get; set; } = false;
     }
 }