C# 模拟http请求网页数据 [网页爬虫]

using System;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using FTE.Framework.Log4NetService;

namespace Proxy.BllServices
{
    /// 

    /// http访问类
    /// 
    public class HttpHelper
    {
        /// 

        /// 访问失败的统一返回字符
        /// 
        public String ErrorReturn { get; private set; } = "HttpHelper access error!";

        /// 

        /// 登录后保存的cookie
        /// 
        private CookieContainer Cookie = new CookieContainer();

        /// 

        /// http post 访问网页
        /// 
        /// 
        /// 
        /// 
        public string HttpPostString(string Url, string postDataStr)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
                request.Method = "POST";
                request.ContentType = "application/x-www-form-urlencoded";
                request.ContentLength = Encoding.UTF8.GetByteCount(postDataStr);
                request.CookieContainer = Cookie;
                Stream myRequestStream = request.GetRequestStream();
                StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));
                myStreamWriter.Write(postDataStr);
                myStreamWriter.Close();

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                response.Cookies = Cookie.GetCookies(response.ResponseUri);

                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                string retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();

                return retString;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http post 网站出错", ex);
            }

            return ErrorReturn;
        }

        public string HttpGet(string Url, string postDataStr)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url + (postDataStr == "" ? "" : "?") + postDataStr);
                request.Method = "GET";
                request.ContentType = "text/html;charset=UTF-8";
                request.CookieContainer = Cookie;

                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, Encoding.GetEncoding("utf-8"));
                string retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();

                return retString;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http get 网站出错", ex);
            }

            return ErrorReturn;
        }

        /// 

        /// 使用form方式post数据[不包含文件]
        /// 
        /// 
        /// 
        /// 
        public string HttpPostForm(string url, NameValueCollection stringDict)
        {
            try
            {
                string responseContent;
                var memStream = new MemoryStream();
                var webRequest = (HttpWebRequest)WebRequest.Create(url);
                // 边界符  
                var boundary = "---------------" + DateTime.Now.Ticks.ToString("x");
                // 边界符  
                var beginBoundary = Encoding.ASCII.GetBytes("--" + boundary + "\r\n");
                // 最后的结束符  
                var endBoundary = Encoding.ASCII.GetBytes("--" + boundary + "--\r\n");

                // 设置属性  
                webRequest.CookieContainer = Cookie;
                webRequest.Method = "POST";
                webRequest.ContentType = "multipart/form-data; boundary=" + boundary;
                // 写入字符串的Key  
                var stringKeyHeader = "\r\n--" + boundary +
                                       "\r\nContent-Disposition: form-data; name=\"{0}\"" +
                                       "\r\n\r\n{1}\r\n";

                foreach (byte[] formitembytes in from string key in stringDict.Keys
                                                 select string.Format(stringKeyHeader, key, stringDict[key])
                                                     into formitem
                                                 select Encoding.UTF8.GetBytes(formitem))
                {
                    memStream.Write(formitembytes, 0, formitembytes.Length);
                }

                // 写入最后的结束边界符  
                memStream.Write(endBoundary, 0, endBoundary.Length);

                webRequest.ContentLength = memStream.Length;

                var requestStream = webRequest.GetRequestStream();

                memStream.Position = 0;
                var tempBuffer = new byte[memStream.Length];
                memStream.Read(tempBuffer, 0, tempBuffer.Length);
                memStream.Close();

                requestStream.Write(tempBuffer, 0, tempBuffer.Length);
                requestStream.Close();

                var httpWebResponse = (HttpWebResponse)webRequest.GetResponse();

                using (var httpStreamReader = new StreamReader(httpWebResponse.GetResponseStream(),
                                                                Encoding.GetEncoding("utf-8")))
                {
                    responseContent = httpStreamReader.ReadToEnd();
                }

                httpWebResponse.Close();
                webRequest.Abort();

                return responseContent;
            }
            catch (Exception ex)
            {
                LoggerManagerSingle.Instance.Error("http form post 网站出错", ex);
            }

            return ErrorReturn;
        }
    }
}

//1.
helper.HttpPostString("http://192.168.1.1/", "luci_username=root&luci_password=password");

//2.
NameValueCollection stringDict = new NameValueCollection();
stringDict.Add("token", token);
stringDict.Add("cbid.wireless.default_radio1.ssid", "everTestWifi");
helper.HttpPostData("http://192.168.1.1/cgi-bin/luci/admin/network/wireless/radio1.network2", stringDict);