I don't think I am getting the full redirect, and not sure if its downloading the web page.

See more:

C#

proxy

WebClient

I have been given a task to create a custom search engine using gcse. I have also been given 300+ websites/domains on a white-list, but due to the sheer strength of our proxy, not all sites on the white-list appear to be accessible. I tried creating a tool to get the redirect of the URL, and tried downloading via downloadData into a byte[] to see if the Uri is accessible. Problem is, our proxy will return dns_unresolved, not found, or proxy blocked. I figured I would just check to see if "DOCTYPE" was found in the document, or terms blocked by proxy "<proxyservername>".

My code:

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Net;
using System.Windows.Forms;

namespace Isitsecure
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
            lstResults.Columns.Add("Host", 285, HorizontalAlignment.Left);
            lstResults.Columns.Add("Redirect", 285, HorizontalAlignment.Left);
            lstResults.Columns.Add("Status", 150, HorizontalAlignment.Left);
        }

        public string[] redirectStrings;
        public string[] listStrings;
        List redirectUrls = new List();

        private void btnLoad_Click(object sender, EventArgs e)
        {
            using (OpenFileDialog ofd = new OpenFileDialog())
            {
                try
                {
                    lstResults.Items.Clear();
                    Array.Clear(listStrings, 0, listStrings.Length);
                    Array.Clear(redirectStrings, 0, redirectStrings.Length);
                    redirectUrls = new List();

                } catch(Exception ex) { }

                ofd.Filter = "txt files (*.txt)|*.txt|All files (*.*)|*.*";
                ofd.FilterIndex = 2;
                ofd.InitialDirectory = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
                ofd.RestoreDirectory = true;

                if (ofd.ShowDialog() == DialogResult.OK)
                {
                    if (ofd.FileName != null)
                    {
                        listStrings = File.ReadAllLines(ofd.FileName); 
                        lblCount.Text = listStrings.Count().ToString();
                    }
                }
            }

            btnLoad.Enabled = false;
            rl.Show();
            bgRedirectFinder.RunWorkerAsync();
        }

        public string GetRedirectUrl(string url)
        {
            try
            {
                HttpWebRequest rqst = (HttpWebRequest)WebRequest.Create(url);
                rqst.AllowAutoRedirect = false;
                rqst.Timeout = 2000;
                rqst.Method = "HEAD";
                HttpWebResponse response = (HttpWebResponse)rqst.GetResponse();

                if (response.StatusCode == HttpStatusCode.Redirect)
                {
                    return response.Headers["Location"];
                }
                else
                {
                    return url;
                }
            }
            catch (WebException we)
            {
                return null;
            }
        }

        public string GetSource(string url)
        {
            using (WebClient wc = new WebClient())
            {
                try
                {
                    byte[] src;
                    string _source = string.Empty;
                    src = wc.DownloadData(url);
                    _source = System.Text.Encoding.ASCII.GetString(src);
                    return _source;
                }
                catch (ArgumentNullException ex)
                {
                    return ex.Message;
                }
                catch (WebException ex)
                {
                    return ex.Message;

                }
                catch (Exception ex)
                {
                    return ex.Message;
                }
                finally
                {
                    //call this if exception occurs or not
                    //in this example, dispose the WebClient
                    wc.Dispose();
                }
            }

        }

        private void btnSubmit_Click(object sender, EventArgs e)
        {
            btnSubmit.Enabled = false;
            bgWork.RunWorkerAsync();
            lblComplete.Visible = true;
        }

        private void bgWork_DoWork(object sender, DoWorkEventArgs e)
        {
            ListViewItem lvi = new ListViewItem();

            try
            {
                for (var s = 0; s  { lvi = lstResults.Items.Add(string.Format("{0}", listStrings[s])); }));
                        lstResults.BeginInvoke(new Action(() => { lvi.SubItems.Add(string.Format("{0}", redirectStrings[s])); }));
                        lstResults.BeginInvoke(new Action(() => { lvi.SubItems.Add(string.Format("PASSED")); }));
                        source = string.Empty;
                    }
                    else
                    {
                        lstResults.BeginInvoke(new Action(() => { lvi = lstResults.Items.Add(string.Format("{0}", listStrings[s])); }));
                        lstResults.BeginInvoke(new Action(() => { lvi.SubItems.Add(string.Format("{0}", redirectStrings[s])); }));
                        lstResults.BeginInvoke(new Action(() => { lvi.SubItems.Add(string.Format("FAIL")); }));
                        source = string.Empty;
                    }

                    int percentage = (s + 1) * 100 / redirectStrings.Length;
                    bgWork.ReportProgress(percentage);
                    this.Invoke(new MethodInvoker(delegate { ResizeListViewColumns(lstResults); }));
                }
            }
            catch (Exception ex)
            {
                MessageBox.Show("An error has occurred: " + ex);
            }
        }

        private void ResizeListViewColumns(ListView lv)
        {
            foreach (ColumnHeader column in lv.Columns)
            {
                if (column.Text != "Redirect" && column.Text != "Host")
                {
                    column.Width = -2;
                }
            }
        }

        private void bgWork_ProgressChanged(object sender, ProgressChangedEventArgs e)
        {
            prgrBar.Value = e.ProgressPercentage;
            lblPerc.Text = string.Format("{0}%", e.ProgressPercentage.ToString());
        }

        private void bgWork_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
        {
            btnSubmit.Enabled = true;
            MessageBox.Show("Scan complete, please export the results to a CSV file", "Scan Complete", MessageBoxButtons.OK, MessageBoxIcon.Information);
            this.Invoke(new MethodInvoker(delegate { lblPosition.Text = string.Empty; }));
            this.Invoke(new MethodInvoker(delegate { lblUrl.Text = string.Empty; }));
            this.Invoke(new MethodInvoker(delegate { lblRedirect.Text = string.Empty; }));
        }

        private void lstResults_VisibleChanged(object sender, EventArgs e)
        {
            ResizeListViewColumns(lstResults);
        }

        private void lblExport_Click(object sender, EventArgs e)
        {
            SaveFileDialog sfd = new SaveFileDialog
            {
                Title = "Choose file to save to",
                FileName = "hosts.csv",
                Filter = "CSV (*.csv)|*.csv",
                FilterIndex = 0,
                InitialDirectory = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments)
            };

            if (sfd.ShowDialog() == DialogResult.OK)
            {

                string[] headers = lstResults.Columns
                           .OfType()
                           .Select(header => header.Text.Trim())
                           .ToArray();

                string[][] items = lstResults.Items
                            .OfType()
                            .Select(lvi => lvi.SubItems
                                .OfType()
                                .Select(si => si.Text).ToArray()).ToArray();

                string table = string.Join(",", headers) + Environment.NewLine;
                foreach (string[] a in items)
                {
                    //a = a_loopVariable;
                    table += string.Join(",", a) + Environment.NewLine;
                }
                table = table.TrimEnd('\r', '\n');
                File.WriteAllText(sfd.FileName, table);
            }
        }

        private void lblExit_Click(object sender, EventArgs e)
        {
            this.Close();
        }

        private void bgRedirectFinder_DoWork(object sender, DoWorkEventArgs e)
        {
            for (int r = 0; r<= listStrings.Length - 1;r++)
            {
                redirectUrls.Add(GetRedirectUrl(listStrings[r]));
                int percent = (r + 1) * 100 / listStrings.Length;
                bgRedirectFinder.ReportProgress(percent);
            }

            redirectStrings = redirectUrls.ToArray();
        }

        Redirect_Locator rl = new Redirect_Locator();

        private void bgRedirectFinder_ProgressChanged(object sender, ProgressChangedEventArgs e)
        {
            
            rl.prgrRedirects.Value = e.ProgressPercentage;
        }

        private void bgRedirectFinder_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
        {
            btnLoad.Enabled = true;
            rl.Hide();
        }
    }
}

Everything comes out clean, but certainly doesn't feel like it runs correctly when I build it and run it.

What I have tried:

I tried checking status, but technically, status returns true or OK on dns_unresolved or proxy blocked, so httpwebrequest.StatusCode does not help. Also typing in fake websites comes back as httpwebrequest.StatusCode == OK. I have resorted to WebCLient from httpwebrequest only because I kept getting status 200 for all sites including the fake sites. I figured I would download the source to see if the source contains the proxy server name when blocked.