网页信息采集核心代码收集

<ol class="dp-c"><li class="alt">1.通过HttpWebResponse 来获取 </li><li>public static string CheckTeamSiteUrl(string url)   </li><li class="alt">    {   </li><li>        string response = "";   </li><li class="alt">        HttpWebResponse httpResponse = null;   </li><li>   </li><li class="alt">        //assert: user have access to URL    </li><li>        try   </li><li class="alt">        {   </li><li>            HttpWebRequest httpRequest = (HttpWebRequest)WebRequest.Create(url);   </li><li class="alt">            httpRequest.Headers.Set("Pragma", "no-cache");   </li><li>   </li><li class="alt">                // request.Headers.Set("KeepAlive", "true");   </li><li>   </li><li class="alt">                httpRequest.CookieContainer = new CookieContainer();   </li><li>   </li><li class="alt">   </li><li>   </li><li class="alt">                httpRequest.Referer = url;   </li><li>   </li><li class="alt">                httpRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)";   </li><li>   </li><li class="alt">                 </li><li>   </li><li class="alt">            httpRequest.Credentials = System.Net.CredentialCache.DefaultCredentials;   </li><li>            httpResponse = (HttpWebResponse)httpRequest.GetResponse();   </li><li class="alt">               </li><li>        }   </li><li class="alt">        catch (Exception ex)   </li><li>        {   </li><li class="alt">            throw new ApplicationException("HTTP 403 Access denied, URL: " + url, ex);   </li><li>        }   </li><li class="alt">   </li><li>        //if here, the URL is correct and the user has access    </li><li class="alt">        try   </li><li>        {   </li><li class="alt">            string strEncod = httpResponse.ContentType;   </li><li>            StreamReader stream;   </li><li class="alt">            if (strEncod.ToLower().IndexOf("utf") != -1)   </li><li>            {   </li><li class="alt">                stream = new StreamReader(httpResponse.GetResponseStream(), System.Text.Encoding.UTF8);   </li><li>            }   </li><li class="alt">            else   </li><li>            {   </li><li class="alt">                stream = new StreamReader(httpResponse.GetResponseStream(), System.Text.Encoding.Default);   </li><li>            }   </li><li class="alt">              </li><li>            char[] buff = new char[4000];   </li><li class="alt">            stream.ReadBlock(buff,0,4000);   </li><li>            response = new string(buff);   </li><li class="alt">            stream.Close();   </li><li>            httpResponse.Close();   </li><li class="alt">        }   </li><li>        catch (Exception ex)   </li><li class="alt">        {   </li><li>            throw new ApplicationException("HTTP 404 Page not found, URL: " + url, ex);   </li><li class="alt">        }   </li><li>        return response;   </li><li class="alt">   </li><li>    }   </li><li class="alt">  </li><li> </li><li>2.通过 WebResponse 来获取 </li><li class="alt"> public static string getPage(String url)   </li><li>    {   </li><li class="alt">        WebResponse result = null;   </li><li>        string resultstring = "";   </li><li class="alt">        try   </li><li>        {   </li><li class="alt">            WebRequest req = WebRequest.Create(url);   </li><li>            req.Timeout = 30000;   </li><li class="alt">            result = req.GetResponse();   </li><li>            Stream ReceiveStream = result.GetResponseStream();   </li><li class="alt">   </li><li>            //read the stream into a string   </li><li class="alt">            //StreamReader sr = new StreamReader(ReceiveStream, System.Text.Encoding.UTF8);   </li><li>            string strEncod = result.ContentType;   </li><li class="alt">            StreamReader sr;   </li><li>            if (strEncod.ToLower().IndexOf("utf") != -1)   </li><li class="alt">            {   </li><li>                sr = new StreamReader(ReceiveStream, System.Text.Encoding.UTF8);   </li><li class="alt">            }   </li><li>            else   </li><li class="alt">            {   </li><li>                sr = new StreamReader(ReceiveStream, System.Text.Encoding.Default);   </li><li class="alt">            }   </li><li>            resultstring = sr.ReadToEnd();   </li><li class="alt">            js.alert(resultstring);   </li><li>            //Console.WriteLine(resultstring);   </li><li class="alt">        }   </li><li>        catch   </li><li class="alt">        {   </li><li>            throw new Exception();   </li><li class="alt">        }   </li><li>        finally   </li><li class="alt">        {   </li><li>            if (result != null)   </li><li class="alt">            {   </li><li>                result.Close();   </li><li class="alt">            }   </li><li>        }   </li><li class="alt">        return resultstring;   </li><li>    }   </li><li class="alt">  </li><li> </li><li>3.通过WebClient来获取 </li><li class="alt"> public string get(int length)   </li><li>    {   </li><li class="alt">        try   </li><li>        {   </li><li class="alt">            getEncodeing();   </li><li>            WebClient wb = new WebClient();   </li><li class="alt">            Stream response = wb.OpenRead(url);   </li><li>            StreamReader reader = new StreamReader(response, this.encoding, true, 256000);   </li><li class="alt">            char[] a = new char[length];   </li><li>            int i  = reader.Read(a,0,length);   </li><li class="alt">            reader.Close();   </li><li>            return new string(a);   </li><li class="alt">        }   </li><li>        catch (Exception e)   </li><li class="alt">        {   </li><li>            return e.Message;   </li><li class="alt">            //return null;   </li><li>        }   </li><li class="alt">    }   </li><li>    private void getEncodeing()   </li><li class="alt">    {   </li><li>        switch (this.encode)   </li><li class="alt">        {   </li><li>            case "UTF-8": encoding = Encoding.UTF8; break;   </li><li class="alt">            case "GB2312": encoding = Encoding.GetEncoding("GB2312"); break;   </li><li>            case "ASCII": encoding = Encoding.ASCII; break;   </li><li class="alt">            default: encoding = Encoding.GetEncoding(encode); break;   </li><li>        }   </li><li class="alt">    }   </li></ol>

FROM http://blog.sina.com.cn/s/blog_758087190100rfri.html

M	T	W	T	F	S	S
« Nov
						1
2	3	4	5	6	7	8
9	10	11	12	13	14	15
16	17	18	19	20	21	22
23	24	25	26	27	28	29
30	31

网页信息采集 核心代码收集

网页信息采集核心代码收集