Add support for accept-charset header.

Author: ioquatix

lib/protocol/http/header/accept_charset.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2020-2023, by Samuel Williams.
+# Copyright, 2023, by Thomas Morgan.
+
+require_relative "split"
+require_relative "quoted_string"
+require_relative "../error"
+
+module Protocol
+	module HTTP
+		module Header
+			# The `accept-charset` header represents a list of character sets that the client can accept.
+			class AcceptCharset < Split
+				ParseError = Class.new(Error)
+				
+				# https://tools.ietf.org/html/rfc7231#section-5.3.1
+				QVALUE = /0(\.[0-9]{0,3})?|1(\.[0]{0,3})?/
+				
+				# https://tools.ietf.org/html/rfc7231#section-5.3.3
+				CHARSETS = /\A(?<charset>#{TOKEN})(;q=(?<q>#{QVALUE}))?\z/
+				
+				Charset = Struct.new(:charset, :q) do
+					def quality_factor
+						(q || 1.0).to_f
+					end
+				end
+				
+				# Parse the `accept-charset` header value into a list of character sets.
+				#
+				# @returns [Array(Charset)] the list of character sets and their associated quality factors.
+				def charsets
+					self.map do |value|
+						if match = value.match(CHARSETS)
+							Charset.new(match[:charset], match[:q])
+						else
+							raise ParseError.new("Could not parse character set: #{value.inspect}")
+						end
+					end
+				end
+				
+				# Sort the character sets by quality factor, with the highest quality factor first.
+				#
+				# @returns [Array(Charset)] the list of character sets sorted by quality factor.
+				def sorted_charsets
+					# We do this to get a stable sort:
+					self.charsets.sort_by.with_index{|object, index| [-object.quality_factor, index]}
+				end
+			end
+		end
+	end
+end

lib/protocol/http/header/quoted_string.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2016-2024, by Samuel Williams.
+
+module Protocol
+	module HTTP
+		module Header
+			# According to https://tools.ietf.org/html/rfc7231#appendix-C
+			TOKEN = /[!#$%&'*+\-.^_`|~0-9A-Z]+/i
+			QUOTED_STRING = /"(?:.(?!(?<!\\)"))*.?"/
+			
+			module QuotedString
+				# Unquote a "quoted-string" value according to https://tools.ietf.org/html/rfc7230#section-3.2.6
+				# It should already match the QUOTED_STRING pattern above by the parser.
+				def self.unquote(value, normalize_whitespace = true)
+					value = value[1...-1]
+					
+					value.gsub!(/\\(.)/, '\1')
+					
+					if normalize_whitespace
+						# LWS = [CRLF] 1*( SP | HT )
+						value.gsub!(/[\r\n]+\s+/, ' ')
+					end
+					
+					return value
+				end
+				
+				# Quote a string if required. Doesn't handle newlines correctly currently.
+				def self.quote(value, force = false)
+					if value =~ /"/ or force
+						"\"#{value.gsub(/["\\]/, "\\\\\\0")}\""
+					else
+						return value
+					end
+				end
+			end
+		end
+	end
+end

lib/protocol/http/headers.rb

@@ -3,6 +3,7 @@
 # Released under the MIT License.
 # Copyright, 2018-2024, by Samuel Williams.
 
+require_relative "header/accept_charset"
 require_relative "header/split"
 require_relative "header/multiple"
 require_relative "header/cookie"
@@ -277,6 +278,8 @@ def []= key, value
 				"last-modified" => Header::Date,
 				"if-modified-since" => Header::Date,
 				"if-unmodified-since" => Header::Date,
+				
+				"accept-charset" => Header::AcceptCharset,
 			}.tap{|hash| hash.default = Split}
 
 			# Delete all header values for the given key, and return the merged value.

test/protocol/http/header/accept_charset.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# Released under the MIT License.
+# Copyright, 2016, by Matthew Kerwin.
+# Copyright, 2017-2024, by Samuel Williams.
+
+require 'protocol/http/header/accept_charset'
+	
+describe Protocol::HTTP::Header::AcceptCharset::Charset do
+	it "should have default quality_factor of 1.0" do
+		charset = subject.new('utf-8', nil)
+		expect(charset.quality_factor).to be == 1.0
+	end
+end
+
+describe Protocol::HTTP::Header::AcceptCharset do
+	let(:header) {subject.new(description)}
+	let(:charsets) {header.sorted_charsets}
+	
+	with "utf-8, iso-8859-1;q=0.5, windows-1252;q=0.25" do
+		it "can parse charsets" do
+			expect(header.length).to be == 3
+			
+			expect(charsets[0].charset).to be == "utf-8"
+			expect(charsets[0].quality_factor).to be == 1.0
+			
+			expect(charsets[1].charset).to be == "iso-8859-1"
+			expect(charsets[1].quality_factor).to be == 0.5
+			
+			expect(charsets[2].charset).to be == "windows-1252"
+			expect(charsets[2].quality_factor).to be == 0.25
+		end
+	end
+	
+	with "windows-1252;q=0.25, iso-8859-1;q=0.5, utf-8" do
+		it "should order based on quality factor" do
+			expect(charsets.collect(&:charset)).to be == %w{utf-8 iso-8859-1 windows-1252}
+		end
+	end
+	
+	with "us-ascii,iso-8859-1;q=0.8,windows-1252;q=0.6,utf-8" do
+		it "should order based on quality factor" do
+			expect(charsets.collect(&:charset)).to be == %w{us-ascii utf-8 iso-8859-1 windows-1252}
+		end
+	end
+	
+	with "*;q=0" do
+		it "should accept wildcard charset" do
+			expect(charsets[0].charset).to be == "*"
+			expect(charsets[0].quality_factor).to be == 0
+		end
+	end
+	
+	with "utf-8, iso-8859-1;q=0.5, windows-1252;q=0.5" do
+		it "should preserve relative order" do
+			expect(charsets[0].charset).to be == "utf-8"
+			expect(charsets[1].charset).to be == "iso-8859-1"
+			expect(charsets[2].charset).to be == "windows-1252"
+		end
+	end
+	
+	it "should not accept invalid input" do
+		bad_values = [
+			# Invalid quality factor:
+			"utf-8;f=1",
+			
+			# Invalid charset separator:
+			"us-ascii;utf-8",
+			
+			# Invalid use of separator:
+			";",
+			
+			# Empty charset (we ignore this one):
+			# ","
+		]
+		
+		bad_values.each do |value|
+			expect{subject.new(value).charsets}.to raise_exception(subject::ParseError)
+		end
+	end
+end