Refactor & update codebase

FlareSolverrSharp.sln

@@ -1,14 +1,16 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 16
-VisualStudioVersion = 16.0.30709.132
+# Visual Studio Version 17
+VisualStudioVersion = 17.11.35312.102
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FlareSolverrSharp", "src\FlareSolverrSharp\FlareSolverrSharp.csproj", "{E541E27A-8D55-4E2F-AC7D-DCA0DCDAC220}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FlareSolverrSharp.Sample", "sample\FlareSolverrSharp.Sample\FlareSolverrSharp.Sample.csproj", "{F44FEFA6-B85B-4C05-AD34-836DF6BF63B9}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FlareSolverrSharp.Tests", "test\FlareSolverrSharp.Tests\FlareSolverrSharp.Tests.csproj", "{89A9D8CB-01BA-43CA-83AE-2D760088154C}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D0184B45-49CD-4C2F-B956-3D2253321FF2}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

sample/FlareSolverrSharp.Sample/ClearanceHandlerSample.cs

@@ -6,68 +6,77 @@
 using System.Threading.Tasks;
 using System.Web;
 
-namespace FlareSolverrSharp.Sample
+namespace FlareSolverrSharp.Sample;
+
+public static class ClearanceHandlerSample
 {
-    public static class ClearanceHandlerSample
-    {
-
-        public static string FlareSolverrUrl = "http://localhost:8191/";
-        public static string ProtectedUrl = "https://badasstorrents.com/torrents/search/720p/date/desc";
-
-        public static async Task SampleGet()
-        {
-            var handler = new ClearanceHandler(FlareSolverrUrl)
-            {
-                MaxTimeout = 60000
-            };
-
-            var client = new HttpClient(handler);
-            var content = await client.GetStringAsync(ProtectedUrl);
-            Console.WriteLine(content);
-        }
-
-        public static async Task SamplePostUrlEncoded()
-        {
-            var handler = new ClearanceHandler(FlareSolverrUrl)
-            {
-                MaxTimeout = 60000
-            };
-
-            var request = new HttpRequestMessage();
-            request.Headers.ExpectContinue = false;
-            request.RequestUri = new Uri(ProtectedUrl);
-            var postData = new Dictionary<string, string> { { "story", "test" }};
-            request.Content = FormUrlEncodedContentWithEncoding(postData, Encoding.UTF8);
-            request.Method = HttpMethod.Post;
-
-            var client = new HttpClient(handler);
-            var content = await client.SendAsync(request);
-            Console.WriteLine(content);
-        }
-
-        static ByteArrayContent FormUrlEncodedContentWithEncoding(
-            IEnumerable<KeyValuePair<string, string>> nameValueCollection, Encoding encoding)
-        {
-            // utf-8 / default
-            if (Encoding.UTF8.Equals(encoding) || encoding == null)
-                return new FormUrlEncodedContent(nameValueCollection);
-
-            // other encodings
-            var builder = new StringBuilder();
-            foreach (var pair in nameValueCollection)
-            {
-                if (builder.Length > 0)
-                    builder.Append('&');
-                builder.Append(HttpUtility.UrlEncode(pair.Key, encoding));
-                builder.Append('=');
-                builder.Append(HttpUtility.UrlEncode(pair.Value, encoding));
-            }
-            // HttpRuleParser.DefaultHttpEncoding == "latin1"
-            var data = Encoding.GetEncoding("latin1").GetBytes(builder.ToString());
-            var content = new ByteArrayContent(data);
-            content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
-            return content;
-        }
-
-    }
+
+	public static string FlareSolverrUrl = "http://localhost:8191/";
+	public static string ProtectedUrl    = "https://badasstorrents.com/torrents/search/720p/date/desc";
+
+	public static async Task SampleGet()
+	{
+		var handler = new ClearanceHandler(FlareSolverrUrl)
+		{
+			Solverr =
+			{
+				MaxTimeout = 60000
+
+			}
+		};
+
+		var client  = new HttpClient(handler);
+		var content = await client.GetStringAsync(ProtectedUrl);
+		Console.WriteLine(content);
+	}
+
+	public static async Task SamplePostUrlEncoded()
+	{
+		var handler = new ClearanceHandler(FlareSolverrUrl)
+		{
+			Solverr =
+			{
+				MaxTimeout = 60000
+
+
+			}
+		};
+
+		var request = new HttpRequestMessage();
+		request.Headers.ExpectContinue = false;
+		request.RequestUri             = new Uri(ProtectedUrl);
+		var postData = new Dictionary<string, string> { { "story", "test" } };
+		request.Content = FormUrlEncodedContentWithEncoding(postData, Encoding.UTF8);
+		request.Method  = HttpMethod.Post;
+
+		var client  = new HttpClient(handler);
+		var content = await client.SendAsync(request);
+		Console.WriteLine(content);
+	}
+
+	static ByteArrayContent FormUrlEncodedContentWithEncoding(
+		IEnumerable<KeyValuePair<string, string>> nameValueCollection, Encoding encoding)
+	{
+		// utf-8 / default
+		if (Encoding.UTF8.Equals(encoding) || encoding == null)
+			return new FormUrlEncodedContent(nameValueCollection);
+
+		// other encodings
+		var builder = new StringBuilder();
+
+		foreach (var pair in nameValueCollection) {
+			if (builder.Length > 0)
+				builder.Append('&');
+			builder.Append(HttpUtility.UrlEncode(pair.Key, encoding));
+			builder.Append('=');
+			builder.Append(HttpUtility.UrlEncode(pair.Value, encoding));
+		}
+
+		// HttpRuleParser.DefaultHttpEncoding == "latin1"
+		var data    = Encoding.GetEncoding("latin1").GetBytes(builder.ToString());
+		var content = new ByteArrayContent(data);
+		content.Headers.ContentType = new MediaTypeHeaderValue("application/x-www-form-urlencoded");
+		return content;
+	}
+
 }

sample/FlareSolverrSharp.Sample/FlareSolverrSharp.Sample.csproj

@@ -2,12 +2,16 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net9.0</TargetFramework>
     <RootNamespace>FlareSolverrSharp.Sample</RootNamespace>
     <AssemblyName>FlareSolverrSharp.Sample</AssemblyName>
-    <Version>3.0.7</Version>
+    <Version>3.0.8</Version>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="JetBrains.Annotations" Version="2024.3.0" />
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="..\..\src\FlareSolverrSharp\FlareSolverrSharp.csproj" />
   </ItemGroup>

sample/FlareSolverrSharp.Sample/Program.cs

@@ -1,12 +1,42 @@
-
-namespace FlareSolverrSharp.Sample
+using System;
+using System.Collections.Generic;
+using FlareSolverrSharp.Exceptions;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+namespace FlareSolverrSharp.Sample;
+
+public static class Program
 {
-    static class Program
-    {
-        static void Main()
-        {
-            ClearanceHandlerSample.SampleGet().Wait();
-            ClearanceHandlerSample.SamplePostUrlEncoded().Wait();
-        }
-    }
-}
+
+	public static async Task Main()
+	{
+		/*ClearanceHandlerSample.SampleGet().Wait();
+		ClearanceHandlerSample.SamplePostUrlEncoded().Wait();*/
+
+		var handler = new ClearanceHandler(Settings.FlareSolverrApiUrl)
+		{
+			EnsureResponseIntegrity = false,
+			Solverr =
+			{
+				MaxTimeout = 60000
+			}
+		};
+
+		var client = new HttpClient(handler);
+
+		HttpRequestMessage[] rg =
+		[
+			new(HttpMethod.Get, "https://ascii2d.net/search/url/https://pomf2.lain.la/f/fy32pj5e.png"),
+			new(HttpMethod.Get, "https://ascii2d.net/search/url/https://i.redd.it/xixxli0axz7b1.jpg"),
+		];
+
+		await Parallel.ForEachAsync(rg, async (x, y) =>
+		{
+			var res = await client.SendAsync(x, y);
+			Console.WriteLine($"{x.RequestUri} -> {res.StatusCode}");
+			return;
+		});
+	}
+
+}

sample/FlareSolverrSharp.Sample/Settings.cs

@@ -0,0 +1,34 @@
+using System;
+using System.Runtime.CompilerServices;
+
+[assembly: InternalsVisibleTo("FlareSolverrSharp.Tests")]
+
+namespace FlareSolverrSharp.Sample;
+
+internal static class Settings
+{
+
+	internal const           string FlareSolverrApiUrl = "http://localhost:8191/";
+	internal const           string ProxyUrl           = "http://127.0.0.1:8888/";
+	internal static readonly Uri    ProtectedUri       = new Uri("https://nowsecure.nl");
+
+	internal static readonly Uri ProtectedPostUri =
+		new Uri("https://badasstorrents.com/torrents/search/720p/date/desc");
+
+	internal static readonly Uri ProtectedDdgUri = new Uri("https://anidex.info/?q=text");
+	internal static readonly Uri ProtectedCcfUri = new Uri("https://www.muziekfabriek.org");
+
+
+	// causes a redirect making the test falsely fail
+	// internal static readonly Uri ProtectedBlockedUri = new Uri("https://cpasbiens3.fr/");
+
+	/*
+	To configure TinyProxy in local:
+	   * sudo vim /etc/tinyproxy/tinyproxy.conf
+	      * edit => LogFile "/tmp/tinyproxy.log"
+	      * edit => Syslog Off
+	   * sudo tinyproxy -d
+	   * sudo tail -f /tmp/tinyproxy.log
+	*/
+
+}

src/FlareSolverrSharp/ChallengeDetector.cs

@@ -3,56 +3,58 @@
 using System.Linq;
 using System.Net;
 using System.Net.Http;
+using System.Threading.Tasks;
+using FlareSolverrSharp.Constants;
 
-namespace FlareSolverrSharp
+namespace FlareSolverrSharp;
+
+public static class ChallengeDetector
 {
-    public static class ChallengeDetector
-    {
-        private static readonly HashSet<string> CloudflareServerNames = new HashSet<string>{
-        "cloudflare",
-        "cloudflare-nginx",
-        "ddos-guard"
-        };
-
-        /// <summary>
-        /// Checks if clearance is required.
-        /// </summary>
-        /// <param name="response">The HttpResponseMessage to check.</param>
-        /// <returns>True if the site requires clearance</returns>
-        public static bool IsClearanceRequired(HttpResponseMessage response) => IsCloudflareProtected(response);
-
-        /// <summary>
-        /// Checks if the site is protected by Cloudflare
-        /// </summary>
-        /// <param name="response">The HttpResponseMessage to check.</param>
-        /// <returns>True if the site is protected</returns>
-        private static bool IsCloudflareProtected(HttpResponseMessage response)
-        {
-            // check response headers
-            if (!response.Headers.Server.Any(i =>
-                    i.Product != null && CloudflareServerNames.Contains(i.Product.Name.ToLower())))
-                return false;
-
-            // detect CloudFlare and DDoS-GUARD
-            if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) ||
-                 response.StatusCode.Equals(HttpStatusCode.Forbidden)) {
-                var responseHtml = response.Content.ReadAsStringAsync().Result;
-                if (responseHtml.Contains("<title>Just a moment...</title>") || // Cloudflare
-                    responseHtml.Contains("<title>Access denied</title>") || // Cloudflare Blocked
-                    responseHtml.Contains("<title>Attention Required! | Cloudflare</title>") || // Cloudflare Blocked
-                    responseHtml.Trim().Equals("error code: 1020") || // Cloudflare Blocked
-                    responseHtml.IndexOf("<title>DDOS-GUARD</title>", StringComparison.OrdinalIgnoreCase) > -1) // DDOS-GUARD
-                    return true;
-            }
-
-            // detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
-            if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" &&
-                response.Content.Headers.ContentEncoding.ToString() == "" &&
-                response.Content.ReadAsStringAsync().Result.ToLower().Contains("ddos"))
-                return true; 
-
-            return false;
-        }
-
-    }
-}
+
+	/// <summary>
+	/// Checks if clearance is required.
+	/// </summary>
+	/// <param name="response">The HttpResponseMessage to check.</param>
+	/// <returns>True if the site requires clearance</returns>
+	public static bool IsClearanceRequiredAsync(HttpResponseMessage response)
+		=> IsCloudflareProtectedAsync(response);
+
+	/// <summary>
+	/// Checks if the site is protected by Cloudflare
+	/// </summary>
+	/// <param name="response">The HttpResponseMessage to check.</param>
+	/// <returns>True if the site is protected</returns>
+	private static bool IsCloudflareProtectedAsync(HttpResponseMessage response)
+	{
+		// check response headers
+		if (response.Headers.Server.Any(i =>
+			                                 i.Product != null
+			                                 && CloudflareValues.CloudflareServerNames.Contains(
+				                                 i.Product.Name.ToLower()))) {
+			// return false;
+			return true;
+		}
+
+		// detect CloudFlare and DDoS-GUARD
+		if (response.StatusCode is HttpStatusCode.ServiceUnavailable or HttpStatusCode.Forbidden
+		    or (HttpStatusCode) CloudflareValues.CloudflareStatusCodes.OriginUnreachable) {
+			var responseHtml = response.Content.ReadAsStringAsync().Result;
+
+
+			if (CloudflareValues.CloudflareBlocked.Any(responseHtml.Contains)      || // Cloudflare Blocked
+			    responseHtml.Trim().StartsWith(CloudflareValues.CLOUDFLARE_ERROR_CODE_PREFIX) || // Cloudflare Blocked
+			    responseHtml.IndexOf(CloudflareValues.DDOS_GUARD_TITLE, StringComparison.OrdinalIgnoreCase)
+			    > -1) // DDOS-GUARD
+				return true;
+		}
+
+		// detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+		if (response.Headers.Vary.ToString()                    == "Accept-Encoding,User-Agent" &&
+		    response.Content.Headers.ContentEncoding.ToString() == String.Empty                 &&
+		    (response.Content.ReadAsStringAsync().Result).ToLower().Contains("ddos"))
+			return true;
+
+		return false;
+	}
+
+}