scraping website with javascript cookie with c#
- by erwin
Hi all,
I want to scrap some things from the following site: http://www.conrad.nl/modelspoor
This is my function:
public string SreenScrape(string urlBase, string urlPath)
{
CookieContainer cookieContainer = new CookieContainer();
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(urlBase + urlPath);
httpWebRequest.CookieContainer = cookieContainer;
httpWebRequest.UserAgent = "Mozilla/6.0 (Windows; U; Windows NT 7.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.9 (.NET CLR 3.5.30729)";
WebResponse webResponse = httpWebRequest.GetResponse();
string result = new System.IO.StreamReader(webResponse.GetResponseStream(), Encoding.Default).ReadToEnd();
webResponse.Close();
if (result.Contains("<frame src="))
{
Regex metaregex = new Regex("http:[a-z:/._0-9!?=A-Z&]*",RegexOptions.Multiline);
result = result.Replace("\r\n", "");
Match m = metaregex.Match(result);
string key = m.Groups[0].Value;
foreach (Match match in metaregex.Matches(result))
{
HttpWebRequest redirectHttpWebRequest = (HttpWebRequest)WebRequest.Create(key);
redirectHttpWebRequest.CookieContainer = cookieContainer;
webResponse = redirectHttpWebRequest.GetResponse();
string redirectResponse = new System.IO.StreamReader(webResponse.GetResponseStream(), Encoding.Default).ReadToEnd();
webResponse.Close();
return redirectResponse;
}
}
return result;
}
But when i do this i get a string with an error from the website that it use javascript.
Does anybody know how to fix this?
Kind regards Erwin
