Merge pull request #54 from klauspost/add-full-lut

Add full lookup table for single rune width.

Author: mattn

benchmark_test.go

@@ -14,19 +14,42 @@ var benchSink int
 func benchRuneWidth(b *testing.B, eastAsianWidth bool, start, stop rune, want int) int {
 	b.Helper()
 	n := 0
-	got := -1
-	c := NewCondition()
-	c.EastAsianWidth = eastAsianWidth
-	for i := 0; i < b.N; i++ {
-		got = n
-		for r := start; r < stop; r++ {
-			n += c.RuneWidth(r)
+	b.Run("regular", func(b *testing.B) {
+		got := -1
+		c := NewCondition()
+		c.EastAsianWidth = eastAsianWidth
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			got = n
+			for r := start; r < stop; r++ {
+				n += c.RuneWidth(r)
+			}
+			got = n - got
 		}
-		got = n - got
-	}
-	if want != 0 && got != want { // some extra checks
-		b.Errorf("got %d, want %d\n", got, want)
-	}
+		if want != 0 && got != want { // some extra checks
+			b.Errorf("got %d, want %d\n", got, want)
+		}
+	})
+	b.Run("lut", func(b *testing.B) {
+		got := -1
+		n = 0
+		c := NewCondition()
+		c.EastAsianWidth = eastAsianWidth
+		c.CreateLUT()
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			got = n
+			for r := start; r < stop; r++ {
+				n += c.RuneWidth(r)
+			}
+			got = n - got
+		}
+		if want != 0 && got != want { // some extra checks
+			b.Errorf("got %d, want %d\n", got, want)
+		}
+	})
 	return n
 }
 func BenchmarkRuneWidthAll(b *testing.B) {
@@ -49,20 +72,44 @@ func BenchmarkRuneWidth768EastAsian(b *testing.B) {
 func benchString1Width(b *testing.B, eastAsianWidth bool, start, stop rune, want int) int {
 	b.Helper()
 	n := 0
-	got := -1
-	c := NewCondition()
-	c.EastAsianWidth = eastAsianWidth
-	for i := 0; i < b.N; i++ {
-		got = n
-		for r := start; r < stop; r++ {
-			s := string(r)
-			n += c.StringWidth(s)
+	b.Run("regular", func(b *testing.B) {
+		got := -1
+		c := NewCondition()
+		c.EastAsianWidth = eastAsianWidth
+		b.ResetTimer()
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			got = n
+			for r := start; r < stop; r++ {
+				s := string(r)
+				n += c.StringWidth(s)
+			}
+			got = n - got
 		}
-		got = n - got
-	}
-	if want != 0 && got != want { // some extra checks
-		b.Errorf("got %d, want %d\n", got, want)
-	}
+		if want != 0 && got != want { // some extra checks
+			b.Errorf("got %d, want %d\n", got, want)
+		}
+	})
+	b.Run("lut", func(b *testing.B) {
+		got := -1
+		n = 0
+		c := NewCondition()
+		c.EastAsianWidth = eastAsianWidth
+		c.CreateLUT()
+		b.ResetTimer()
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			got = n
+			for r := start; r < stop; r++ {
+				s := string(r)
+				n += c.StringWidth(s)
+			}
+			got = n - got
+		}
+		if want != 0 && got != want { // some extra checks
+			b.Errorf("got %d, want %d\n", got, want)
+		}
+	})
 	return n
 }
 func BenchmarkString1WidthAll(b *testing.B) {

runewidth.go

@@ -34,7 +34,13 @@ func handleEnv() {
 		EastAsianWidth = env == "1"
 	}
 	// update DefaultCondition
-	DefaultCondition.EastAsianWidth = EastAsianWidth
+	if DefaultCondition.EastAsianWidth != EastAsianWidth {
+		DefaultCondition.EastAsianWidth = EastAsianWidth
+		if len(DefaultCondition.combinedLut) > 0 {
+			DefaultCondition.combinedLut = DefaultCondition.combinedLut[:0]
+			CreateLUT()
+		}
+	}
 }
 
 type interval struct {
@@ -89,6 +95,7 @@ var nonprint = table{
 
 // Condition have flag EastAsianWidth whether the current locale is CJK or not.
 type Condition struct {
+	combinedLut        []byte
 	EastAsianWidth     bool
 	StrictEmojiNeutral bool
 }
@@ -104,10 +111,16 @@ func NewCondition() *Condition {
 // RuneWidth returns the number of cells in r.
 // See http://www.unicode.org/reports/tr11/
 func (c *Condition) RuneWidth(r rune) int {
+	if r < 0 || r > 0x10FFFF {
+		return 0
+	}
+	if len(c.combinedLut) > 0 {
+		return int(c.combinedLut[r>>1]>>(uint(r&1)*4)) & 3
+	}
 	// optimized version, verified by TestRuneWidthChecksums()
 	if !c.EastAsianWidth {
 		switch {
-		case r < 0x20 || r > 0x10FFFF:
+		case r < 0x20:
 			return 0
 		case (r >= 0x7F && r <= 0x9F) || r == 0xAD: // nonprint
 			return 0
@@ -124,7 +137,7 @@ func (c *Condition) RuneWidth(r rune) int {
 		}
 	} else {
 		switch {
-		case r < 0 || r > 0x10FFFF || inTables(r, nonprint, combining):
+		case inTables(r, nonprint, combining):
 			return 0
 		case inTable(r, narrow):
 			return 1
@@ -138,6 +151,27 @@ func (c *Condition) RuneWidth(r rune) int {
 	}
 }
 
+// CreateLUT will create an in-memory lookup table of 557056 bytes for faster operation.
+// This should not be called concurrently with other operations on c.
+// If options in c is changed, CreateLUT should be called again.
+func (c *Condition) CreateLUT() {
+	const max = 0x110000
+	lut := c.combinedLut
+	if len(c.combinedLut) != 0 {
+		// Remove so we don't use it.
+		c.combinedLut = nil
+	} else {
+		lut = make([]byte, max/2)
+	}
+	for i := range lut {
+		i32 := int32(i * 2)
+		x0 := c.RuneWidth(i32)
+		x1 := c.RuneWidth(i32 + 1)
+		lut[i] = uint8(x0) | uint8(x1)<<4
+	}
+	c.combinedLut = lut
+}
+
 // StringWidth return width as you can see
 func (c *Condition) StringWidth(s string) (width int) {
 	g := uniseg.NewGraphemes(s)
@@ -271,3 +305,12 @@ func FillLeft(s string, w int) string {
 func FillRight(s string, w int) string {
 	return DefaultCondition.FillRight(s, w)
 }
+
+// CreateLUT will create an in-memory lookup table of 557055 bytes for faster operation.
+// This should not be called concurrently with other operations.
+func CreateLUT() {
+	if len(DefaultCondition.combinedLut) > 0 {
+		return
+	}
+	DefaultCondition.CreateLUT()
+}

runewidth_test.go

@@ -1,3 +1,4 @@
+//go:build !js && !appengine
 // +build !js,!appengine
 
 package runewidth
@@ -87,7 +88,56 @@ func TestRuneWidthChecksums(t *testing.T) {
 			t.Errorf("TestRuneWidthChecksums = %s,\n\tsha256 = %s want %s",
 				testcase.name, gotSHA, testcase.wantSHA)
 		}
+
+		// Test with LUT
+		c.CreateLUT()
+		for r := rune(0); r <= utf8.MaxRune; r++ {
+			buf[r] = byte(c.RuneWidth(r))
+		}
+		gotSHA = fmt.Sprintf("%x", sha256.Sum256(buf))
+		if gotSHA != testcase.wantSHA {
+			t.Errorf("TestRuneWidthChecksums = %s,\n\tsha256 = %s want %s",
+				testcase.name, gotSHA, testcase.wantSHA)
+		}
+	}
+}
+
+func TestDefaultLUT(t *testing.T) {
+	var testcases = []struct {
+		name           string
+		eastAsianWidth bool
+		wantSHA        string
+	}{
+		{"ea-no", false, "4eb632b105d3b2c800dda9141381d0b8a95250a3a5c7f1a5ca2c4d4daaa85234"},
+		{"ea-yes", true, "c2ddc3bdf42d81d4c23050e21eda46eb639b38b15322d35e8eb6c26f3b83ce92"},
+	}
+
+	old := os.Getenv("RUNEWIDTH_EASTASIAN")
+	defer os.Setenv("RUNEWIDTH_EASTASIAN", old)
+
+	CreateLUT()
+	for _, testcase := range testcases {
+		c := DefaultCondition
+
+		if testcase.eastAsianWidth {
+			os.Setenv("RUNEWIDTH_EASTASIAN", "1")
+		} else {
+			os.Setenv("RUNEWIDTH_EASTASIAN", "0")
+		}
+		handleEnv()
+
+		buf := make([]byte, utf8.MaxRune+1)
+		for r := rune(0); r <= utf8.MaxRune; r++ {
+			buf[r] = byte(c.RuneWidth(r))
+		}
+		gotSHA := fmt.Sprintf("%x", sha256.Sum256(buf))
+		if gotSHA != testcase.wantSHA {
+			t.Errorf("TestRuneWidthChecksums = %s,\n\tsha256 = %s want %s",
+				testcase.name, gotSHA, testcase.wantSHA)
+		}
 	}
+	// Remove for other tests.
+	DefaultCondition.combinedLut = nil
 }
 
 func checkInterval(first, last rune) bool {