xiugai

FCS.Common/Helper/CommonHelper.cs

@@ -546,7 +546,6 @@ namespace FCS.Common
         public static string GetHtmlString_jb(string url, string title = "", int timeout = 90 * 1000, bool isWebSoxket = false, string webProxy = "", string method = "get", int notIpNUmber = 100)
         {
             var NotIpList = new List<string>();
-            var ip = webProxy.IsEmpty() ? GetIp() : webProxy;
             var html = new HttpHelper().GetHtml(new HttpItem
             {
                 Url = url,
@@ -567,8 +566,11 @@ namespace FCS.Common
                     || (html.Html.Contains("301 Moved Permanently") && html.Html.ToLower().Contains("moved permanently"))
                     || (html.Html.Contains("The requested URL could not be retrieved") && html.Html.ToLower().Contains("could not be retrieved"))
                     || html.Html.Contains("缓存访问被拒绝")
+                    || html.Html.Contains("操作太频繁了，请先歇一歇")
                     || (title.IsEmpty() && !html.Html.Contains(title)))
                 {
+                    if(html.Html.Contains("操作太频繁了，请先歇一歇"))
+                        Thread.Sleep(1000);
                     number++;
                     if (number > 40)
                         return "";
@@ -577,38 +579,11 @@ namespace FCS.Common
                         ConfigurationManager.AppSettings["Termination"].ToString();
                         break;
                     }
-                    NotIpList.Add(ip);
-                    if (NotIpList.Distinct().ToList().Count == IpCount)
-                    {
-                        //IAsyncResult asyncResult;
-                        //lock (locker)
-                        //{
-                        //    GetIPDataBYOne task = new GetIPDataBYOne(IPHelper.GetIPDataBYOne);
-                        //    asyncResult = task.BeginInvoke(new List<string> { url }, title, false, null, null);
-
-                        //    while (asyncResult != null && !asyncResult.AsyncWaitHandle.WaitOne(100, false))
-                        //    {
-
-                        //    }
-                        //    ip = task.EndInvoke(asyncResult);
-                        //    if (ip.IsEmpty())
-                        //    {
-                        return ConfigurationManager.AppSettings["Termination"].ToString();
-                        // break;
-                        //  }
-                        //}
-                        //}
-                    }
-                    else
-                        ip = webProxy.IsEmpty() ? GetIp() : webProxy;
 
-                    while (NotIpList.Contains(ip))
-                        ip = webProxy.IsEmpty() ? GetIp() : webProxy;
                     html = new HttpHelper().GetHtml(new HttpItem
                     {
                         Url = url,
                         Method = method,
-                        WebProxy = new WebProxy(ip)
                     });
                 }
             }

FCS.Crawler/Content/Jobs/BasketJobs/Basketball/B_TeamandPlayerCBAJob.cs

@@ -131,7 +131,7 @@ namespace FCS.Crawler.Basketball
                     services.SqlBulkCopyAdd<B_Players>(PlayerListnew);
                 Trace.WriteLine("插入B_Players 完结 数目：" + PlayerListnew.Count);
                 if (listid.Count > 0)
-                    services.Delete<B_Players>(" and Id in (" + string.Join("','", listid) + ")");
+                    services.Delete<B_Players>(" and Id in ('" + string.Join("','", listid) + "')");
                 Trace.WriteLine("删除 B_Players 数目：" + listid.Count);
             }
 
@@ -281,7 +281,7 @@ namespace FCS.Crawler.Basketball
             Trace.WriteLine("插入 B_TeamStatistics 完结 数目：" + stalist_add.Count);
 
             if (listid.Count > 0)
-                services.Delete<B_TeamStatistics>(" and Id in (" + string.Join("','", listid) + ")");
+                services.Delete<B_TeamStatistics>(" and Id in ('" + string.Join("','", listid) + "')");
             Trace.WriteLine("删除 B_TeamStatistics 数目：" + listid.Count);
 
         }
@@ -295,32 +295,32 @@ namespace FCS.Crawler.Basketball
             var AllTeamList = services.Query<B_Team>((" and EventId='{0}' and Remark is not null").FormatMe(eventId)).ToList();
             //获取分组存取的url ds.Tables[0].Rows.Count
 
-            //var teamlist_new=GetTeamHtml(Teamurl);
-            //Trace.WriteLine("获取B_Team 完结 数目："+ teamlist_new.Length);
-            //List<B_Players> PlayerList = new List<B_Players>();
-            //List<B_Team> teamlist = new List<B_Team>();
-            //for (int i = 0; i < teamlist_new.Length; i++)
-            //{
-            //    var teamorplayer = GetPlayersHtml(teamlist_new[i]);
-            //    if (teamorplayer.team != null)
-            //        teamlist.Add(teamorplayer.team);
-            //    if (teamorplayer.playerlist.Count > 0)
-            //    {
-            //        foreach (var item in teamorplayer.playerlist)
-            //        {
-            //            item.TeamId = teamorplayer.team.Id;
-            //            PlayerList.Add(item);
-            //        }
-                    
-            //    }
-                  
-            //}
-            //if (teamlist != null)
-            //    services.SqlBulkCopyAdd<B_Team>(teamlist);
-            //Trace.WriteLine("插入B_Team 完结 数目：" + teamlist.Count);
-            //if (PlayerList!=null)
-            //    services.SqlBulkCopyAdd<B_Players>(PlayerList);
-            //Trace.WriteLine("插入B_Players 完结 数目：" + PlayerList.Count);
+            var teamlist_new = GetTeamHtml(Teamurl);
+            Trace.WriteLine("获取B_Team 完结 数目：" + teamlist_new.Length);
+            List<B_Players> PlayerList = new List<B_Players>();
+            List<B_Team> teamlist = new List<B_Team>();
+            for (int i = 0; i < teamlist_new.Length; i++)
+            {
+                var teamorplayer = GetPlayersHtml(teamlist_new[i]);
+                if (teamorplayer.team != null)
+                    teamlist.Add(teamorplayer.team);
+                if (teamorplayer.playerlist.Count > 0)
+                {
+                    foreach (var item in teamorplayer.playerlist)
+                    {
+                        item.TeamId = teamorplayer.team.Id;
+                        PlayerList.Add(item);
+                    }
+
+                }
+
+            }
+            if (teamlist != null)
+                services.SqlBulkCopyAdd<B_Team>(teamlist);
+            Trace.WriteLine("插入B_Team 完结 数目：" + teamlist.Count);
+            if (PlayerList != null)
+                services.SqlBulkCopyAdd<B_Players>(PlayerList);
+            Trace.WriteLine("插入B_Players 完结 数目：" + PlayerList.Count);
 
             //球队统计
             AllTeamList = services.Query<B_Team>((" and EventId='{0}' and Remark is not null").FormatMe(eventId)).ToList();
@@ -449,6 +449,8 @@ namespace FCS.Crawler.Basketball
         public string[][] GetTeamHtml(string url)
         {
             var html = CommonHelper.GetHtmlString_jb(url, "",1000*100);
+            if (html.Contains("操作太频繁了，请先歇一歇"))
+                Trace.WriteLine("球队队员信息：" + html);
             if (html.IsEmpty())
                 return null;
             Regex r = new Regex(@"var arrTeam = (\w|\W)*?]];");
@@ -469,6 +471,8 @@ namespace FCS.Crawler.Basketball
         {
             teamorplayer detail = new teamorplayer();
             var html = CommonHelper.GetHtmlString_jb(TeamDetailurl.FormatMe(data[0]), "", 1000 * 100);
+            if (html.Contains("操作太频繁了，请先歇一歇"))
+                Trace.WriteLine("球队队员信息：" + html);
             if (html.IsEmpty())
                 return detail;
             Regex r = new Regex(@"var teamSummary =(\w|\W)*?];");
@@ -507,7 +511,7 @@ namespace FCS.Crawler.Basketball
             {
                 var jsonstring =("["+ item.Value.TrimStart(',','[').Replace("]]","]").Replace("[", "\"").Replace("]", "\"") + "]").JsonToEntity<string[]>();
                 //球员
-                Thread.Sleep(2000);
+                //Thread.Sleep(2000);
                 html = CommonHelper.GetHtmlString_jb(playerDetailurl.FormatMe(jsonstring[0]), "", 1000 * 100);
                 if(html.Contains("操作太频繁了，请先歇一歇"))
                     Trace.WriteLine("球队数据：" + html);

FCS.Crawler/Content/Jobs/FootJobs/ZCLotteryGames/GamesJob.cs

@@ -230,8 +230,8 @@ namespace FCS.Crawler.ZCLotteryGames
                         services.Delete<F_Games>(" and PK in (" + string.Join(",", listint1) + ")");
                     }
                 }
-                //gamesList = new List<F_Games>();
-                //gamesList_new = new List<F_Games>();
+                gamesList = new List<F_Games>();
+                gamesList_new = new List<F_Games>();
                 return gamesList_new1;
             }
             catch (Exception e)

FCS.Crawler/Program.cs

@@ -30,9 +30,9 @@ namespace FCS.Crawler
         {
             //JobManage job = new JobManage();
             //job.JobStart();
-            //B_TeamandPlayerCBAJob job = new B_TeamandPlayerCBAJob();
+            B_TeamandPlayerCBAJob job = new B_TeamandPlayerCBAJob();
             //job.GetAll();
-            //job.Click();
+            job.Click();
             ServiceBase[] ServicesToRun;
             ServicesToRun = new ServiceBase[]
             {