I've been trying to automate a log in to a website I frequent, www.bungie.net. The site is associated with Microsoft and Xbox Live, and as such makes uses of the Windows Live ID API when people log in to their site.
I am relatively new to creating web spiders/robots, and I worry that I'm misunderstanding some of the most basic concepts. I've simulated logins to other sites such as Facebook and Gmail, but live.com has given me nothing but trouble.
Anyways, I've been using Wireshark and the Firefox addon Tamper Data to try and figure out what I need to post, and what cookies I need to include with my requests. As far as I know these are the steps one must follow to log in to this site.
1. Visit https: //login.live.com/login.srf?wa=wsignin1.0&rpsnv=11&ct=1268167141&rver=5.5.4177.0&wp=LBI&wreply=http:%2F%2Fwww.bungie.net%2FDefault.aspx&id=42917
2. Recieve the cookies MSPRequ and MSPOK.
3. Post the values from the form ID "PPSX", the values from the form ID "PPFT", your username, your password all to a changing URL similar to: https: //login.live.com/ppsecure/post.srf?wa=wsignin1.0&rpsnv=11&ct=
(there are a few numbers that change at the end of that URL)
4. Live.com returns the user a page with more hidden forms to post. The client then posts the values from the form "ANON", the value from the form "ANONExp" and the values from the form "t" to the URL: http ://www.bung ie.net/Default.aspx?wa=wsignin1.0
5. After posting that data, the user is returned a variety of cookies the most important of which is "BNGAuth" which is the log in cookie for the site.
Where I am having trouble is on fifth step, but that doesn't neccesarily mean I've done all the other steps correctly. I post the data from "ANON", "ANONExp" and "t" but instead of being returned a BNGAuth cookie, I'm returned a cookie named "RSPMaybe" and redirected to the home page.
When I review the Wireshark log, I noticed something that instantly stood out to me as different between the log when I logged in with Firefox and when my program ran. It could be nothing but I'll include the picture here for you to review. I'm being returned an HTTP packet from the site before I post the data in the fourth step. I'm not sure how this is happening, but it must be a side effect from something I'm doing wrong in the HTTPS steps.
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;
using System.Security.Cryptography;
using System.Security.Cryptography.X509Certificates;
using System.Web;
namespace SpiderFromScratch
{
class Program
{
static void Main(string[] args)
{
CookieContainer cookies = new CookieContainer();
Uri url = new Uri("https://login.live.com/login.srf?wa=wsignin1.0&rpsnv=11&ct=1268167141&rver=5.5.4177.0&wp=LBI&wreply=http:%2F%2Fwww.bungie.net%2FDefault.aspx&id=42917");
HttpWebRequest http = (HttpWebRequest)HttpWebRequest.Create(url);
http.Timeout = 30000;
http.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8 (.NET CLR 3.5.30729)";
http.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
http.Headers.Add("Accept-Language", "en-us,en;q=0.5");
http.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
http.Headers.Add("Keep-Alive", "300");
http.Referer = "http://www.bungie.net/";
http.ContentType = "application/x-www-form-urlencoded";
http.CookieContainer = new CookieContainer();
http.Method = WebRequestMethods.Http.Get;
HttpWebResponse response = (HttpWebResponse)http.GetResponse();
StreamReader readStream = new StreamReader(response.GetResponseStream());
string HTML = readStream.ReadToEnd();
readStream.Close();
//gets the cookies (they are set in the eighth header)
string[] strCookies = response.Headers.GetValues(8);
response.Close();
string name, value;
Cookie manualCookie;
for (int i = 0; i < strCookies.Length; i++)
{
name = strCookies[i].Substring(0, strCookies[i].IndexOf("="));
value = strCookies[i].Substring(strCookies[i].IndexOf("=") + 1, strCookies[i].IndexOf(";") - strCookies[i].IndexOf("=") - 1);
manualCookie = new Cookie(name, "\"" + value + "\"");
Uri manualURL = new Uri("http://login.live.com");
http.CookieContainer.Add(manualURL, manualCookie);
}
//stores the cookies to be used later
cookies = http.CookieContainer;
//Get the PPSX value
string PPSX = HTML.Remove(0, HTML.IndexOf("PPSX"));
PPSX = PPSX.Remove(0, PPSX.IndexOf("value") + 7);
PPSX = PPSX.Substring(0, PPSX.IndexOf("\""));
//Get this random PPFT value
string PPFT = HTML.Remove(0, HTML.IndexOf("PPFT"));
PPFT = PPFT.Remove(0, PPFT.IndexOf("value") + 7);
PPFT = PPFT.Substring(0, PPFT.IndexOf("\""));
//Get the random URL you POST to
string POSTURL = HTML.Remove(0, HTML.IndexOf("https://login.live.com/ppsecure/post.srf?wa=wsignin1.0&rpsnv=11&ct="));
POSTURL = POSTURL.Substring(0, POSTURL.IndexOf("\""));
//POST with cookies
http = (HttpWebRequest)HttpWebRequest.Create(POSTURL);
http.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8 (.NET CLR 3.5.30729)";
http.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
http.Headers.Add("Accept-Language", "en-us,en;q=0.5");
http.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
http.Headers.Add("Keep-Alive", "300");
http.CookieContainer = cookies;
http.Referer = "https://login.live.com/login.srf?wa=wsignin1.0&rpsnv=11&ct=1268158321&rver=5.5.4177.0&wp=LBI&wreply=http:%2F%2Fwww.bungie.net%2FDefault.aspx&id=42917";
http.ContentType = "application/x-www-form-urlencoded";
http.Method = WebRequestMethods.Http.Post;
Stream ostream = http.GetRequestStream();
//used to convert strings into bytes
System.Text.ASCIIEncoding encoding = new System.Text.ASCIIEncoding();
//Post information
byte[] buffer = encoding.GetBytes("PPSX=" + PPSX +"&PwdPad=IfYouAreReadingThisYouHaveTooMuc&login=YOUREMAILGOESHERE&passwd=YOURWORDGOESHERE" +
"&LoginOptions=2&PPFT=" + PPFT);
ostream.Write(buffer, 0, buffer.Length);
ostream.Close();
HttpWebResponse response2 = (HttpWebResponse)http.GetResponse();
readStream = new StreamReader(response2.GetResponseStream());
HTML = readStream.ReadToEnd();
response2.Close();
ostream.Dispose();
foreach (Cookie cookie in response2.Cookies)
{
Console.WriteLine(cookie.Name + ": ");
Console.WriteLine(cookie.Value);
Console.WriteLine(cookie.Expires);
Console.WriteLine();
}
//SET POSTURL value
string POSTANON = "http://www.bungie.net/Default.aspx?wa=wsignin1.0";
//Get the ANON value
string ANON = HTML.Remove(0, HTML.IndexOf("ANON"));
ANON = ANON.Remove(0, ANON.IndexOf("value") + 7);
ANON = ANON.Substring(0, ANON.IndexOf("\""));
ANON = HttpUtility.UrlEncode(ANON);
//Get the ANONExp value
string ANONExp = HTML.Remove(0, HTML.IndexOf("ANONExp"));
ANONExp = ANONExp.Remove(0, ANONExp.IndexOf("value") + 7);
ANONExp = ANONExp.Substring(0, ANONExp.IndexOf("\""));
ANONExp = HttpUtility.UrlEncode(ANONExp);
//Get the t value
string t = HTML.Remove(0, HTML.IndexOf("id=\"t\""));
t = t.Remove(0, t.IndexOf("value") + 7);
t = t.Substring(0, t.IndexOf("\""));
t = HttpUtility.UrlEncode(t);
//POST the Info and Accept the Bungie Cookies
http = (HttpWebRequest)HttpWebRequest.Create(POSTANON);
http.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.8) Gecko/20100202 Firefox/3.5.8 (.NET CLR 3.5.30729)";
http.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
http.Headers.Add("Accept-Language", "en-us,en;q=0.5");
http.Headers.Add("Accept-Encoding", "gzip,deflate");
http.Headers.Add("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
http.Headers.Add("Keep-Alive", "115");
http.CookieContainer = new CookieContainer();
http.ContentType = "application/x-www-form-urlencoded";
http.Method = WebRequestMethods.Http.Post;
http.Expect = null;
ostream = http.GetRequestStream();
int test = ANON.Length;
int test1 = ANONExp.Length;
int test2 = t.Length;
buffer = encoding.GetBytes("ANON=" + ANON +"&ANONExp=" + ANONExp + "&t=" + t);
ostream.Write(buffer, 0, buffer.Length);
ostream.Close();
//Here lies the problem, I am not returned the correct cookies.
HttpWebResponse response3 = (HttpWebResponse)http.GetResponse();
GZipStream gzip = new GZipStream(response3.GetResponseStream(), CompressionMode.Decompress);
readStream = new StreamReader(gzip);
HTML = readStream.ReadToEnd();
//gets both cookies
string[] strCookies2 = response3.Headers.GetValues(11);
response3.Close();
}
}
}
This has given me problems and I've put many hours into learning about HTTP protocols so any help would be appreciated. If there is an article detailing a similar log in to live.com feel free to point the way. I've been looking far and wide for any articles with working solutions.
If I could be clearer, feel free to ask as this is my first time using Stack Overflow.
