FCS/FCS.Crawler/Content/Jobs/FootJobs/ZCLotteryTeam/FootBallTeamsJob.cs

using FCS.Common;
using FCS.Interface;
using FCS.Models;
using HtmlAgilityPack;
using Quartz;
using System;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

namespace FCS.Crawler.ZCLotteryTeam
{
    /// <summary>
    /// 抓取球队信息
    /// </summary>
    public class FootBallTeamsJob : CommonJob, IJob
    {
        private static List<F_Team> teamList = new List<F_Team>();
        private static List<F_Team> qhyyteamList = new List<F_Team>();
        private static List<F_Team> allTeamList = new List<F_Team>();
        public FootBallTeamsJob()
        {
            log = new LogHelper();
            services = IOC.Resolve<IDTOpenCode>();
        }

        public void Execute(IJobExecutionContext context)
        {
            Config = CommonHelper.GetConfigFromDataMap(context.JobDetail.JobDataMap);
            GetAll();
        }

        public void GetAll()
        {
            ThreadPool.SetMinThreads(10, 10);
            ThreadPool.SetMaxThreads(200, 200);
            allTeamList = services.GetTeamList();
            var ds = SqlHelper.ExecuteDataset(CommandType.Text, GetAllEventUrl);
            if (ds != null && ds.Tables.Count > 0)
            {
                //获取赛事存取的url ds.Tables[0].Rows.Count
                for (int i = 0; i < ds.Tables[0].Rows.Count; i++)
                {
                    var url = ds.Tables[0].Rows[i]["Remark"].ToString();
                    var eventId = ds.Tables[0].Rows[i]["Id"].ToString();
                    var eventName = ds.Tables[0].Rows[i]["Name"].ToString();
                    Task.Run(() =>
                    {
                        var mainUrl = "http://saishi.zgzcw.com" + url;

                        Stopwatch sw = new Stopwatch();
                        Trace.WriteLine("开始：" + mainUrl);
                        sw.Start();

                        HtmlDocument doc = CommonHelper.GetHtml(mainUrl, "足彩");
                        var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='select_options']/a");
                        //循环赛季
                        if (TongJiListData != null)//&& !doc.DocumentNode.InnerText.Contains("球会友谊")
                        {
                            foreach (var item in TongJiListData)
                            {
                                var sjurl = item.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
                                sjurl = sjurl.Replace("－", "-");
                                GetEventHtml(sjurl, eventId, eventName, doc.DocumentNode.InnerText.Contains("球会友谊"));
                            }
                            sw.Stop();
                            Trace.WriteLine(Thread.CurrentThread.ManagedThreadId.ToString() + "||" + sw.ElapsedTicks);
                        }

                    });

                }
                int maxWorkerThreads, workerThreads;
                int maxportThreads, portThreads;

                while (true)
                {
                    /*
                     GetAvailableThreads()：检索由 GetMaxThreads 返回的线程池线程的最大数目和当前活动数目之间的差值。
                     而GetMaxThreads 检索可以同时处于活动状态的线程池请求的数目。
                     通过最大数目减可用数目就可以得到当前活动线程的数目，如果为零，那就说明没有活动线程，说明所有线程运行完毕。
                     */
                    ThreadPool.GetMaxThreads(out maxWorkerThreads, out maxportThreads);
                    ThreadPool.GetAvailableThreads(out workerThreads, out portThreads);
                    Thread.Sleep(1000);

                    Trace.WriteLine("正在执行任务的线程数" + (maxWorkerThreads - workerThreads));
                    if (maxWorkerThreads - workerThreads == 0)
                    {
                        Console.WriteLine("Thread Finished!");
                        break;
                    }
                }
                try
                {
                    if (teamList != null)
                    {
                        foreach (var item in teamList)
                        {
                            if (allTeamList.Where(o => o.Name == item.Name).Count() == 0)
                            {
                                services.AddTeam(FCSLottery.F_Team, item);
                            }
                        }
                    }
                    if (qhyyteamList != null)
                    {
                        foreach (var item in qhyyteamList)
                        {
                            if (allTeamList.Where(o => o.Name == item.Name).Count() == 0)
                            {
                                services.AddTeam(FCSLottery.F_Team, item);
                            }
                        }
                        // TODO球会友谊球队解析
                    }
                }
                catch (Exception ex)
                {
                    throw;

                }
            }
        }
        /// <summary>
        /// 抓取每个赛事下面的球队信息   
        /// </summary>
        /// <param name="url"></param>
        /// <param name="eventId"></param>
        /// <param name="eventName"></param>
        public bool GetEventHtml(string url, string eventId, string eventName, bool IsQHYY)
        {
            if (url.Contains("wwaattssuunn"))
            {
                return false;
            }
            var mainUrl = url;
            Task.Run(() =>
            {
                HtmlDocument doc = CommonHelper.GetHtml(mainUrl, "足彩");

                //获取左边的列表
                var TongJiListData = doc.DocumentNode.SelectNodes("//*[@class='tongji_list']");
                if (TongJiListData != null)
                {
                    foreach (var item in TongJiListData)
                    {
                        if (item.InnerText.Contains("球队列表") && item.InnerHtml != null)
                        {
                            //获取当前赛事的球队列表
                            HtmlDocument teamHtml = new HtmlDocument();
                            teamHtml.LoadHtml(item.InnerHtml);
                            var teams = teamHtml.DocumentNode.SelectNodes("//ul/a");
                            if (teams != null)
                            {
                                Task.Run(() =>
                                {
                                    //球会友谊这个赛事里面的球队太多，单独提取，先读取出数据，后面插入到数据库再进行解析球队
                                    if (IsQHYY)
                                    {
                                        foreach (var teamitem in teams)
                                        {
                                            int index = teams.IndexOf(item);
                                            if (teamitem.InnerHtml != null && teamitem.InnerHtml != "")
                                            {
                                                var teamhref = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
                                                var teamName = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")).Value;

                                                teamhref = teamhref.Replace("－", "-");

                                                var listcont = qhyyteamList.ToList().Where(o => o.Name.Trim() == teamName.Trim()).Count();
                                                if (listcont == 0)
                                                {
                                                    F_Team f_Team1 = new F_Team();
                                                    f_Team1.Id = Guid.NewGuid().ToString();
                                                    f_Team1.Name = teamName;
                                                    f_Team1.EventId = eventId;
                                                    f_Team1.Remark = teamhref;
                                                    qhyyteamList.Add(f_Team1);
                                                }
                                            }
                                        }
                                    }
                                    else
                                    {
                                        foreach (var teamitem in teams)
                                        {
                                            int index = teams.IndexOf(item);
                                            if (teamitem.InnerHtml != null && teamitem.InnerHtml != "")
                                            {
                                                var teamhref = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
                                                var teamName = teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")) == null ? "" : teamitem.Attributes.SingleOrDefault(a => a.Name.Equals("title")).Value;

                                                teamhref = teamhref.Replace("－", "-");

                                                var listcont = teamList.ToList().Where(o => o.Name.Trim() == teamName.Trim()).Count();
                                                if (listcont == 0)
                                                {
                                                    F_Team f_Team1 = new F_Team();
                                                    f_Team1.Id = Guid.NewGuid().ToString();
                                                    f_Team1.Name = teamName;
                                                    f_Team1.EventId = eventId;
                                                    f_Team1.Remark = url;
                                                    teamList.Add(f_Team1);
                                                }

                                                if (teamhref != "")
                                                {
                                                    GetTeamHtml(teamhref, teamName, eventId);
                                                }
                                            }
                                        }
                                    }
                                });
                            }
                        }
                    }
                }
            });
            return true;
        }

        /// <summary>
        /// 读取球队的详细信息页
        /// </summary>
        /// <param name="url"></param>
        /// <param name="TeamName"></param>
        public bool GetTeamHtml(string url, string TeamName, string eventId)
        {
            var list = teamList.ToList().Where(o => o.Name== TeamName);
            if (list == null)
            {
                return false;
            }
            var listcont = list.Count();
            HtmlDocument doc = CommonHelper.GetHtml(url, "足彩");
            var count = 0;
            while (doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']") == null && count < 3)
            {
                count++;
                doc = CommonHelper.GetHtml(url, "足彩");
            }
            if (doc == null || doc.DocumentNode.InnerText.Contains("球队还没有资料"))
            {
                F_Team f_Team1 = new F_Team();
                f_Team1.Id = Guid.NewGuid().ToString();
                f_Team1.Name = TeamName;
                teamList.Add(f_Team1);
                return false;
            }

            //球队图片
            var teamImg = doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dt/img");

            //球队介绍
            var teamIntroData = doc.DocumentNode.SelectNodes("//*[@class='introduceDiv']").FirstOrDefault().InnerHtml.Trim();

            //球队的基本信息
            var teamData = doc.DocumentNode.SelectNodes("//*[@class='star_dl']/dd").ToList();
            F_Team f_Team = new F_Team();
            f_Team.Id = Guid.NewGuid().ToString();
            f_Team.Name = TeamName;
            f_Team.EventId = eventId;
            f_Team.Describe = teamIntroData;
            f_Team.Remark = url;
            f_Team.LogoImage = teamImg.FirstOrDefault().Attributes.SingleOrDefault(a => a.Name.Equals("src")).Value;
            if (teamData != null)
            {
                var contry = teamData.Where(o => o.InnerText.Contains("国家")).FirstOrDefault().InnerText.Replace("\t", "").Split('：');
                f_Team.Address = contry.Count() > 1 ? contry[1].Trim() : "";

                var setUpTime = teamData.Where(o => o.InnerText.Contains("球队成立")).FirstOrDefault().InnerText.Replace("\t", "").Split('：');
                f_Team.SetUpDateTime = setUpTime[1];// (setUpTime.Count() > 1 && setUpTime[1] != "") ? DateTime.Parse(realTime(setUpTime[1])) : DateTime.Parse("1500-01-01");

                var coath = teamData.Where(o => o.InnerText.Contains("教练")).FirstOrDefault().InnerText.Replace("\t", "").Split('：');
                f_Team.Coach = coath.Count() > 1 ? coath[1].Trim() : "";

                var city = teamData.Where(o => o.InnerText.Contains("城市")).FirstOrDefault().InnerText.Replace("\t", "").Split('：');
                f_Team.CityName = city.Count() > 1 ? city[1].Trim() : "";

                var vence = teamData.Where(o => o.InnerText.Contains("球场")).FirstOrDefault().InnerText.Replace("\t", "").Split('：');
                f_Team.Venue = vence.Count() > 1 ? vence[1].Trim() : "";

                var website = teamData.Where(o => o.InnerText.Contains("官网")).FirstOrDefault().InnerHtml;
                HtmlDocument websitedoc = new HtmlDocument();
                websitedoc.LoadHtml(website);
                var webData = websitedoc.DocumentNode.SelectNodes("//var/a");
                f_Team.Website = webData.FirstOrDefault().Attributes.SingleOrDefault(a => a.Name.Equals("href")).Value;
            }

            teamList.Add(f_Team);
            return true;
        }


        public string realTime(string time)
        {
            time = time.Replace(":", "-").Replace("：", "-");
            DateTime outvalue = new DateTime();
            if (DateTime.TryParse(time, out outvalue))
            {
                return time;
            }
            var splittime = time.Split('-');
            if (splittime.Count() == 1)
            {
                return time + "-01-01";
            }
            if (splittime.Count() == 2)
            {
                return time + "-01";
            }

            return "1500-01-01";
        }


        #region SQL语句

        private static string GetAllEventUrl = @"select Id,Name, Remark from F_Events where Remark is not null  ";

        #endregion



    }
}