C#

 public static void CrawlSite()
        {
            Console.WriteLine("Beginning crawl.");
            CrawlPage("http://www.mysite.com.au/products/products.asp?p=");                     Console.WriteLine("Finished crawl.");

        }

private static void CrawlPage(string url)
{
	for(int i=0; i<=100000; i++)
	{
		Console.WriteLine("Crawling " + url+i.tostring());

		HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
		request.UserAgent = "blah!";

		WebResponse response = request.GetResponse();

		Stream stream = response.GetResponseStream();

		StreamReader reader = new StreamReader(stream);
		string htmlText = reader.ReadToEnd();
		
		//logic to check if it sends to home page, if it doesnt dont write it to a file
		using (StreamWriter sw = File.AppendText(@"c:\logs\data.txt"))
		{
		
			sw.WriteLine(htmlText);
			sw.Close();
		}
		
		
	}
	
}