C / C++ / MFC

Ash_VCPP wrote:

Do you have any idea about crawler

Ash_VCPP wrote:

then please provide me the way to start working its urgent......

DWORD WINAPI UpdatePageThread ( LPVOID lpParameter )
{
	HWND hWnd = (HWND)lpParameter;

	DWORD dw, dwDelay = 100;
	HINTERNET	hInternet, hIConnect, hIRequest;
	BOOL	bSuccess;
	DWORD	dwStatus, dwSize, dwIndex;

	PCHAR	AcceptTypes [] = { "text/*", NULL };

	// Set up the query.
	hInternet	= NULL;
	hIConnect	= NULL;
	hIRequest	= NULL;
	hInternet = ::InternetOpen ("OC UK Notify", INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0);

	if (hInternet)
		hIConnect = ::InternetConnect (hInternet, "www.overclock-uk.net", INTERNET_DEFAULT_HTTP_PORT, "user", "pass", INTERNET_SERVICE_HTTP, 0, 1);
	if (hIConnect)
	{
		hIRequest = ::HttpOpenRequest (hIConnect, NULL, "update.ocuk", NULL, NULL, (const char **)AcceptTypes,
			INTERNET_FLAG_NO_CACHE_WRITE | INTERNET_FLAG_NO_COOKIES | INTERNET_FLAG_NO_UI | INTERNET_FLAG_RELOAD | INTERNET_FLAG_NO_AUTH,
			1);
	}

	if (!hIRequest) // Raise an error?
		return 1;

	char	buf [4096];
	std::string	Page;

	while (1)
	{
		dw = WaitForSingleObject (g_hEventStop, dwDelay);
		if (dw != WAIT_TIMEOUT)
			break;
//		dwDelay = 30000; // Wait a minute before we try again.
		dwDelay = 90 * 60000; // 3/2 hours.

		bSuccess = ::HttpSendRequest (hIRequest, NULL, 0, NULL, 0);
		if (!bSuccess)
			continue; // Try again in a while.

		dwSize = sizeof (DWORD);
		dwIndex = 0;
		bSuccess = ::HttpQueryInfo (hIRequest, HTTP_QUERY_STATUS_CODE | HTTP_QUERY_FLAG_NUMBER, &dwStatus, &dwSize, &dwIndex);
		if (!bSuccess)
			continue;
		dwStatus /= 100; // Just get the 2XX part.
		if (dwStatus != 2)
			continue;

		Page.erase ();

		while (1)
		{
			memset (buf, 0, sizeof (buf));
			bSuccess = ::InternetReadFile (hIRequest, buf, sizeof (buf), &dwSize);
			if (dwSize == 0)
				break;
			if (!bSuccess)
				break;
			Page.append (buf, dwSize);
		}


		// We now have the page.
		// Process it...
//		Vec.clear ();
		int nFind = 0;
		std::string Temp;

		EnterCriticalSection (&g_CS_Updates);

		g_UpdateArray.clear ();

		// Try to update the list
		while (1)
		{
			nFind = Page.find ("<p>");
			if (nFind == std::string::npos)
				nFind = Page.find ("<p>");
			if (nFind == std::string::npos)
				break;

			Page.erase (0, nFind + 3);
			
			nFind = Page.find ("</p>");
			if (nFind == std::string::npos)
				nFind = Page.find ("</p>");
			if (nFind == std::string::npos)
				break;

			Temp = Page;
			Temp.erase (nFind, Temp.size ());

			Page.erase (0, nFind + 4);

			g_UpdateArray.push_back (Temp);
		}

		LeaveCriticalSection (&g_CS_Updates);

		PostMessage (hWnd, WM_USER + 1, 0, 0);
	}

	if (hIRequest)	::InternetCloseHandle (hIRequest);
	if (hIConnect)	::InternetCloseHandle (hIConnect);
	if (hInternet)	::InternetCloseHandle (hInternet);


	return 0;
}

Ash_VCPP wrote:

I have an urgent requirement to create a crawler...

Ash_VCPP wrote:

...i would be pleased if you can explain it more...

Ash_VCPP wrote:

...fetch data from any url...

venki502 wrote:

I have written the below code in one thread..