Mamimum page fetch with maximum bandwith
- by Ehsan
Hi
I want to create an application like a spider I've implement fetching page as the following code in multi-thread application but there is two problem
1) I want to use my maximum bandwidth to send/receive request, how should I config my request to do so (Like Download Accelerator application and the like) cause I heard the normal application will use 66% of the available bandwidth.
2) I don't know what exactly HttpWebRequest.KeepAlive do, but as its name implies I think i can create a connection to a website and without closing the connection sending another request to that web site using existing connection. does it boost performance or Im wrong
public PageFetchResult Fetch()
{
PageFetchResult fetchResult = new PageFetchResult();
try
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(URLAddress);
HttpWebResponse resp = (HttpWebResponse)req.GetResponse();
Uri requestedURI = new Uri(URLAddress);
Uri responseURI = resp.ResponseUri;
if (Uri.Equals(requestedURI, responseURI))
{
string resultHTML = "";
byte[] reqHTML = ResponseAsBytes(resp);
if (!string.IsNullOrEmpty(FetchingEncoding))
resultHTML = Encoding.GetEncoding(FetchingEncoding).GetString(reqHTML);
else if (!string.IsNullOrEmpty(resp.CharacterSet))
resultHTML = Encoding.GetEncoding(resp.CharacterSet).GetString(reqHTML);
req.Abort();
resp.Close();
fetchResult.IsOK = true;
fetchResult.ResultHTML = resultHTML;
}
else
{
URLAddress = responseURI.AbsoluteUri;
relayPageCount++;
if (relayPageCount > 5)
{
fetchResult.IsOK = false;
fetchResult.ErrorMessage = "Maximum page redirection occured.";
relayPageCount = 0;
return fetchResult;
}
req.Abort();
resp.Close();
return Fetch();
}
}
catch (Exception ex)
{
fetchResult.IsOK = false;
fetchResult.ErrorMessage = ex.Message;
}
return fetchResult;
}
