Making WebRequests in parallel...
Under the assumption that making sequential WebRequests is slower than making them in parallel i wrote a little program that returns the HTTP status code for each URI in a list. Because the number of WaitHandles on a system is limited to 64 and i would have been required to hack around this limitation i decided to use ThreadPool instead…
using System;
using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Net;
namespace ManyRequests
{
class Program
{
static void Main(string[] args)
{
List<uri> uris = new List<uri>();
uris.Add(new Uri("http://www.timvw.be"));
uris.Add(new Uri("http://example.com/does\_not\_exist"));
uris.Add(new Uri("http://www.timvw.be/c-sharp"));
uris.Add(new Uri("http://www.timvw.be/rss-feed/"));
uris.Add(new Uri("http://localhost"));
Console.WriteLine("Getting the HttpStatusCodes...");
HttpStatusCodeReader httpStatusCodeReader = new HttpStatusCodeReader(uris);
int[] httpStatusCodes = httpStatusCodeReader.GetHttpStatusCodes();
for (int i = 0; i < uris.Count; ++i)
{
Console.WriteLine("{0} {1}", httpStatusCodes[i], uris[i]); }
Console.Write("{0}Press any key to continue…", Environment.NewLine);
Console.ReadKey();
}
}
public class HttpStatusCodeReader
{
private List<uri> uris;
private int[] httpStatusCodes;
private object syncLock;
private int completed;
public HttpStatusCodeReader(List<uri> uris)
{
if (uris == null)
{
throw new ArgumentNullException("uris");
}
foreach (Uri uri in uris)
{
if (uri.Scheme != Uri.UriSchemeHttp && uri.Scheme != Uri.UriSchemeHttps)
{
throw new ArgumentException(uri.ToString() + " is not valid http(s) uri.", "uris");
}
}
this.uris = uris;
this.httpStatusCodes = new int[uris.Count];
this.syncLock = new object();
this.completed = 0;
}
public int[] GetHttpStatusCodes()
{
for (int i = 0; i < this.httpStatusCodes.Length; ++i)
{
HttpWebRequest httpWebRequest = WebRequest.Create(this.uris[i]) as HttpWebRequest;
httpWebRequest.Method = "HEAD";
httpWebRequest.AllowAutoRedirect = true;
httpWebRequest.BeginGetResponse(this.GetResponseCompleted, new object[] { httpWebRequest, i });
}
lock (this.syncLock)
{
while (this.completed < this.httpStatusCodes.Length)
{
Monitor.Wait(this.syncLock);
}
}
return this.httpStatusCodes;
}
private void GetResponseCompleted(IAsyncResult ar)
{
object[] objects = ar.AsyncState as object[];
HttpWebRequest httpWebRequest = objects[0] as HttpWebRequest;
int index = (int)objects[1];
HttpWebResponse httpWebResponse = null;
try
{
httpWebResponse = httpWebRequest.EndGetResponse(ar) as HttpWebResponse;
this.httpStatusCodes[index] = (int)httpWebResponse.StatusCode;
}
catch (WebException webException)
{
httpWebResponse = webException.Response as HttpWebResponse;
if (httpWebResponse != null)
{
this.httpStatusCodes[index] = (int)httpWebResponse.StatusCode;
}
}
finally
{
if (httpWebResponse != null)
{
httpWebResponse.Close();
}
lock (this.syncLock)
{
Interlocked.Add(ref this.completed, 1);
Monitor.Pulse(this.syncLock);
}
}
}
}
}