C#登录后携带cookie爬取数据

前一段时间，公司以前的一个数据采集任务突然之间采集下来的数据都是0了，也就是未登录状态能够获取到的数据，于是猜想肯定是网站的服务升级了，升级了数据接口的逻辑，于是便开始解决此问题。

此采集程序是由.net core开发，采用Quartz定时任务定时采集数据。

下面是解决方法：

1.首先从登录url拿到cookie，然后保存至内存中

2.打开需要爬取的网页时，将拿到的cookie放到打开网页的request中

3.在第2步中ContentType 和UserAgent 需要与登录获取cookie一致

        /// <summary>
        /// 获取Cookie
        /// </summary>
        /// <param name="url">登录url,例如http://www.website.com/user/doLogin</param>
        /// <param name="params">登录参数，例如username=username&pwd=pwd&forever=1</param>
        /// <returns></returns>
        public static CookieCollection GetCookieCollection(string url, string params)
        {
            CookieContainer cc = new CookieContainer();
            ASCIIEncoding encoding = new ASCIIEncoding();
            byte[] data = encoding.GetBytes(params);

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            request.ContentLength = data.Length;
            request.UserAgent = "Chrome/87.0.4280.66"; 

            Stream newStream = request.GetRequestStream();
            newStream.Write(data, 0, data.Length);
            newStream.Close();

            request.CookieContainer = cc;
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            CookieCollection cookieCollection = response.Cookies;
            DateTime dt = DateTime.Now.AddMinutes(365 * 24 * 60);//为cookie添加过期时间
            for (int i = 0; i < cookieCollection.Count; i++)
            {
                cookieCollection[i].Expires = dt;
            }

            return cookieCollection;
        }

        /// <summary>
        /// 获取网页代码
        /// </summary>
        /// <param name="url">需要爬取的数据所在网页url</param>
        /// <param name="cookieCollection">GetCookieCollection方法获取到的cookie</param>
        /// <returns></returns>
        public static string GetWebContent(string url, CookieCollection cookieCollection)                     {
            CookieContainer cc = new CookieContainer();
            cc.Add(cookieCollection);
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);

            request.CookieContainer = cc;
            request.ContentType = "application/x-www-form-urlencoded";//与登录一致
            request.UserAgent = "Chrome/87.0.4280.66"; //与登录一致
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            Stream stream = response.GetResponseStream();
            string webContent = new StreamReader(stream, System.Text.Encoding.Default).ReadToEnd();
            return webContent;
        }