Operation has timed out error when making many request simultaneously

See more:

C#

WinForms

C#4.0

WebMethod

I have a set of N different URLs as input to my program. I need to check the status code of each of them and generate a report. I am using HttpWebRequest-HttpWebResponse classes of .Net.

Also, if I get 5XX status codes I give 'X' times retry attempts to the particular URLs.

The problem I am facing is when I test my program against huge number of URLs say 30-40, for first 3-4 inputs program works properly but then I get error "The request was aborted. Operation has timed out" or "Unable to connect Remote Server" and then again say after 10-15 inputs, for the last 4-5 inputs i get proper response. But when I fire the same URLs which gives time out error in browser, it gets opened.

I tried with many googled solutions like, closed the Response object, aborted the request, disposed/closed all stream objects, set ServicePointManager's Connection Limit to 500, set timeout property of request to 3 mins, but all in Vain nothing solved it.

I really don't know what is the real issue behind it.

If anyone can help me out with this, it will be a very great help from you. Kindly suggest me some solution.

Sample Code:

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;

namespace WindowsFormsApplication5
{
    class Class1
    {
        private void processStart()
        {
            string line = "";
            int UrlCnt = -1;
            string InputPath = @"C:\\MyFolder\Input.txt";
            using (StreamReader oStreamReader = new StreamReader(InputPath))
            {

                while ((line = oStreamReader.ReadLine()) != null)
                {
                    UrlCnt++;
                    string URL = line;

                    GetStatusofURLs(URL, false, -1);
                }

                oStreamReader.Close();
                oStreamReader.Dispose();
            }
        }
        public void GetStatusofURLs(string URL, bool IsCalledforRetry, int RetryCount)
        {
            string startTime = "";
            string timeOut = "30";
            string endTime = "";
            bool IsAttemptSuccess = false;
            bool IsSuccessOnRetry = false;
            bool ReachedMaxTrial = false;
            startTime = DateTime.Now.ToString("hh:mm:ss.fff");
            HttpWebRequest oReq = (HttpWebRequest)WebRequest.Create(URL);
            WebRequest.DefaultWebProxy = null;
            oReq.Timeout = Convert.ToInt32(timeOut) * 1000;
            oReq.AllowAutoRedirect = false;
            ServicePointManager.DefaultConnectionLimit = 500;
            try
            {

                using ((HttpWebResponse)oReq.GetResponse())
                {
                    HttpWebResponse oRes = (HttpWebResponse)oReq.GetResponse();
                    endTime = DateTime.Now.ToString("yyyyMMddHHmmssffff");

                    IsAttemptSuccess = true;

                    if (IsCalledforRetry)
                        IsSuccessOnRetry = true;

                    if ((int)oRes.StatusCode >= 300 && (int)oRes.StatusCode < 600)
                        IsAttemptSuccess = false;
                    oRes.Close();
                    oReq.Abort();
                }
            }
            catch (Exception ex)
            {
                if (ex is WebException)
                {
                    WebException we = (WebException)ex;
                    if (we != null)
                    {
                        endTime = System.DateTime.Now.ToString("hh:mm:ss.fff");
                        if (((HttpWebResponse)we.Response) != null)
                        {
                            HttpWebResponse oRes = ((HttpWebResponse)we.Response);
                            if(!IsCalledforRetry)
                            {
                            if ((int)oRes.StatusCode >= 500)
                                RetryforSuccess(URL, ref IsSuccessOnRetry, ref  ReachedMaxTrial);
}
                        }
                    }
                    else if (!ReachedMaxTrial && !IsCalledforRetry)
                        RetryforSuccess(URL, ref  IsSuccessOnRetry, ref  ReachedMaxTrial);
                }
            }
        }
        public void RetryforSuccess(string URL, ref bool IsSuccessOnRetry, ref bool ReachedMaxTrial)
        {
            for (int i = 1; i < 5; i++)
            {
                if (IsSuccessOnRetry)
                    break;
                if (i == 5)
                {
                    ReachedMaxTrial = true;
                    GetStatusofURLs(URL, true, i);
                    break;
                }
                else
                    GetStatusofURLs(URL, true, i);
            }
        }
    }
}