feat: support Unicode line editing

pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.alibaba.middleware</groupId>
   <artifactId>termd-core</artifactId>
-  <version>1.1.7.15</version>
+  <version>1.1.7.16-SNAPSHOT</version>
   <packaging>jar</packaging>
   <modelVersion>4.0.0</modelVersion>
 

src/main/java/io/termd/core/http/HttpTtyConnection.java

@@ -130,7 +130,7 @@ public void writeToDecoder(String msg) {
       if ("read".equals(action)) {
           lastAccessedTime = System.currentTimeMillis();
           String data = obj.getString("data");
-          decoder.write(data.getBytes()); //write back echo
+          decoder.write(data.getBytes(charset)); // 按连接字符集进入统一解码链路
       } else if ("resize".equals(action)) {
           try {
               int cols = obj.containsKey("cols") ? obj.getIntValue("cols") : size.x();

src/main/java/io/termd/core/readline/LineBuffer.java

@@ -96,8 +96,8 @@ public LineBuffer insert(int cp) {
       if (cp != '\n') {
         throw new IllegalArgumentException("LineBuffer can only contain \\n control char");
       }
-    } else if (w != 1) {
-      throw new IllegalArgumentException("LineBuffer cannot contain chars of width!=1 for the moment");
+    } else if (cp == 0) {
+      throw new IllegalArgumentException("LineBuffer cannot contain the null control char");
     }
     if (cursor < size) {
       System.arraycopy(data, cursor, data, cursor + 1, size - cursor);
@@ -303,13 +303,123 @@ private int findEndOfLine(int offset) {
   }
 
   public void update(LineBuffer dst, Consumer<int[]> out, int width) {
-    new Update(out, width).perform(dst);
+    if (containsNonSingleWidth() || dst.containsNonSingleWidth()) {
+      new Redraw(out, width).perform(dst);
+    } else {
+      new Update(out, width).perform(dst);
+    }
+  }
+
+  private boolean containsNonSingleWidth() {
+    for (int i = 0; i < size; i++) {
+      int codePoint = data[i];
+      if (codePoint != '\n' && Wcwidth.of(codePoint) != 1) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * 非单宽字符采用整行重绘。终端显示的基本单位是单元格，整行重绘可以直接处理宽字符和组合字符，
+   * 也能在删除组合字符时清除已经附着在基础字符上的旧字形。
+   */
+  private class Redraw {
+
+    private final Consumer<int[]> out;
+    private final int width;
+    private int scrCol;
+    private int scrRow;
+
+    private Redraw(Consumer<int[]> out, int width) {
+      this.out = out;
+      this.width = width;
+      Vector position = getCursorPosition(width);
+      this.scrCol = position.x();
+      this.scrRow = position.y();
+    }
+
+    private void perform(LineBuffer dst) {
+      moveCursor(0, 0);
+
+      int lastSourceRow = getPosition(size, width).y();
+      for (int row = 0; row <= lastSourceRow; row++) {
+        out.accept(new int[]{'\033', '[', 'K'});
+        if (row < lastSourceRow) {
+          out.accept(new int[]{'\033', '[', '1', 'B'});
+          scrRow++;
+        }
+      }
+      moveCursor(0, 0);
+
+      if (dst.size > 0) {
+        out.accept(dst.toArray());
+      }
+      Vector end = dst.getPosition(dst.size, width);
+      scrCol = end.x();
+      scrRow = end.y();
+
+      if (dst.endsAtRightMargin(width)) {
+        out.accept(new int[]{' ', '\r'});
+        scrCol = 0;
+      }
+
+      Vector cursorPosition = dst.getCursorPosition(width);
+      moveCursor(cursorPosition.x(), cursorPosition.y());
+
+      data = dst.data.clone();
+      cursor = dst.cursor;
+      size = dst.size;
+    }
+
+    private void moveCursor(int col, int row) {
+      if (scrCol != col) {
+        if (col == 0) {
+          out.accept(new int[]{'\r'});
+          scrCol = 0;
+        } else {
+          while (scrCol != col) {
+            if (scrCol < col) {
+              scrCol++;
+              out.accept(new int[]{'\033', '[', '1', 'C'});
+            } else {
+              scrCol--;
+              out.accept(new int[]{'\b'});
+            }
+          }
+        }
+      }
+      while (scrRow != row) {
+        if (row < scrRow) {
+          scrRow--;
+          out.accept(new int[]{'\033', '[', '1', 'A'});
+        } else {
+          scrRow++;
+          out.accept(new int[]{'\033', '[', '1', 'B'});
+        }
+      }
+    }
+  }
+
+  private boolean endsAtRightMargin(int width) {
+    if (getPosition(size, width).x() != 0) {
+      return false;
+    }
+    for (int i = size - 1; i >= 0; i--) {
+      int codePoint = data[i];
+      if (codePoint == '\n') {
+        return false;
+      }
+      if (Wcwidth.of(codePoint) > 0) {
+        return true;
+      }
+    }
+    return false;
   }
 
   // The update algorithm encapsulated in an inner class
   // todo : use term capabilities instead of hardcoded ansi programming
   // todo : support other control chars
-  // todo : support codepoint of with != 1 (like combining chars, etc...)
   // todo : issue existing chars for moving right instead of cursor left movement
   private class Update {
 

src/main/java/io/termd/core/readline/Readline.java

@@ -291,9 +291,10 @@ void resize(int oldWith, int newWidth) {
 
       // Erase screen
       LineBuffer abc = new LineBuffer(buffer.getCapacity());
-      abc.insert(currentPrompt);
+      int[] promptCodePoints = Helper.toCodePoints(currentPrompt);
+      abc.insert(promptCodePoints);
       abc.insert(buffer.toArray());
-      abc.setCursor(currentPrompt.length() + buffer.getCursor());
+      abc.setCursor(promptCodePoints.length + buffer.getCursor());
 
       // Recompute new cursor
       Vector pos = abc.getCursorPosition(newWidth);
@@ -371,9 +372,10 @@ public Vector size() {
      */
     public void redraw() {
       LineBuffer toto = new LineBuffer(buffer.getCapacity());
-      toto.insert(Helper.toCodePoints(currentPrompt));
+      int[] promptCodePoints = Helper.toCodePoints(currentPrompt);
+      toto.insert(promptCodePoints);
       toto.insert(buffer.toArray());
-      toto.setCursor(currentPrompt.length() + buffer.getCursor());
+      toto.setCursor(promptCodePoints.length + buffer.getCursor());
       LineBuffer abc = new LineBuffer(toto.getCapacity());
       abc.update(toto, conn.stdoutHandler(), size.x());
     }
@@ -391,13 +393,14 @@ public Interaction refresh(LineBuffer buffer) {
     private void refresh(LineBuffer update, int width) {
       LineBuffer copy3 = new LineBuffer(update.getCapacity());
       final List<Integer> codePoints = new LinkedList<Integer>();
-      copy3.insert(Helper.toCodePoints(currentPrompt));
+      int[] promptCodePoints = Helper.toCodePoints(currentPrompt);
+      copy3.insert(promptCodePoints);
       copy3.insert(buffer().toArray());
-      copy3.setCursor(currentPrompt.length() + buffer().getCursor());
+      copy3.setCursor(promptCodePoints.length + buffer().getCursor());
       LineBuffer copy2 = new LineBuffer(copy3.getCapacity());
-      copy2.insert(Helper.toCodePoints(currentPrompt));
+      copy2.insert(promptCodePoints);
       copy2.insert(update.toArray());
-      copy2.setCursor(currentPrompt.length() + update.getCursor());
+      copy2.setCursor(promptCodePoints.length + update.getCursor());
       copy3.update(copy2, new Consumer<int[]>() {
         @Override
         public void accept(int[] data) {

src/test/java/io/termd/core/http/HttpTtyConnectionTest.java

@@ -0,0 +1,51 @@
+package io.termd.core.http;
+
+import io.termd.core.function.Consumer;
+import io.termd.core.util.Helper;
+import io.termd.core.util.Vector;
+import org.junit.Test;
+
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+
+public class HttpTtyConnectionTest {
+
+  @Test
+  public void testTextInputUsesConnectionCharset() {
+    final List<Integer> actual = new ArrayList<Integer>();
+    HttpTtyConnection connection = new HttpTtyConnection(
+        Charset.forName("UTF-16BE"), new Vector(80, 24)) {
+      @Override
+      protected void write(byte[] buffer) {
+      }
+
+      @Override
+      public void execute(Runnable task) {
+        task.run();
+      }
+
+      @Override
+      public void schedule(Runnable task, long delay, TimeUnit unit) {
+        task.run();
+      }
+
+      @Override
+      public void close() {
+      }
+    };
+    connection.setStdinHandler(new Consumer<int[]>() {
+      @Override
+      public void accept(int[] codePoints) {
+        actual.addAll(Helper.list(codePoints));
+      }
+    });
+
+    connection.writeToDecoder("{\"action\":\"read\",\"data\":\"中文\"}");
+
+    assertEquals(Helper.list(Helper.toCodePoints("中文")), actual);
+  }
+}

src/test/java/io/termd/core/io/BinaryEncodingTest.java

@@ -23,6 +23,7 @@ public class BinaryEncodingTest {
   public void testChars() throws IOException {
     testChars("A", 65);
     testChars("\u20AC", -30, -126, -84); // Euro
+    testChars("中文", -28, -72, -83, -26, -106, -121);
     testChars(new StringBuilder().appendCodePoint(66231).toString(), -16, -112, -118, -73); // Surrogate
   }
 

src/test/java/io/termd/core/readline/LineBufferTest.java

@@ -229,7 +229,7 @@ public void testCursorInvisibleChar() {
     assertEquals(new Vector(0, 2), buffer.getCursorPosition(1));
   }
 
-  // @Test
+  @Test
   public void testCursorPositionWithMultiCell1() {
     LineBuffer buffer = new LineBuffer();
     buffer.insert('한', 'b');
@@ -243,7 +243,7 @@ public void testCursorPositionWithMultiCell1() {
     }
   }
 
-  // @Test
+  @Test
   public void testCursorPositionWithMultiCell2() {
     LineBuffer buffer = new LineBuffer();
     buffer.insert('a', '한');

src/test/java/io/termd/core/readline/ReadlineTest.java

@@ -10,6 +10,7 @@
 
 import java.util.Collections;
 import java.util.LinkedList;
+import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * @author <a href="mailto:julien@julienviet.com">Julien Viet</a>
@@ -263,6 +264,99 @@ public void testBuffering() {
     term.assertAt(2, 0);
   }
 
+  @Test
+  public void testChineseInput() {
+    TestTerm term = new TestTerm(this);
+    Supplier<String> line = term.readlineComplete();
+    term.read('中', '文', '\r');
+    assertEquals("中文", line.get());
+    assertEquals(0, term.getBellCount());
+  }
+
+  @Test
+  public void testChineseEditing() {
+    TestTerm term = new TestTerm(this);
+    Supplier<String> line = term.readlineComplete();
+    term.read('A', '中', 'B');
+    term.assertScreen("% A中B");
+    term.assertAt(0, 6);
+
+    term.read(BACKWARD_KEY);
+    term.assertAt(0, 5);
+    term.read(BACKWARD_DELETE_KEY);
+    term.assertScreen("% AB");
+    term.assertAt(0, 3);
+
+    term.read('\r');
+    assertEquals("AB", line.get());
+  }
+
+  @Test
+  public void testChineseWrapsBeforeLastCell() {
+    TestTerm term = new TestTerm(this).setWidth(5);
+    term.readlineFail();
+    term.read('A', 'B', '中', 'C');
+    term.assertScreen("% AB", "中C");
+    term.assertAt(1, 3);
+
+    term.read(BACKWARD_KEY);
+    term.assertAt(1, 2);
+    term.read(BACKWARD_KEY);
+    term.assertAt(0, 4);
+  }
+
+  @Test
+  public void testSupplementaryUnicodePrompt() {
+    TestTerm term = new TestTerm(this);
+    final AtomicReference<String> line = new AtomicReference<String>();
+    String prompt = new String(Character.toChars(0x20000)) + "> ";
+    term.readline.readline(term.conn, prompt, new Consumer<String>() {
+      @Override
+      public void accept(String value) {
+        line.set(value);
+      }
+    });
+
+    term.read('A', 'B');
+    term.assertScreen(prompt + "AB");
+    term.assertAt(0, 6);
+    term.read(BACKWARD_KEY);
+    term.assertAt(0, 5);
+    term.read('\r');
+    assertEquals("AB", line.get());
+  }
+
+  @Test
+  public void testCombiningCharacterEditing() {
+    TestTerm term = new TestTerm(this);
+    Supplier<String> line = term.readlineComplete();
+    term.read('e', '\u0301', 'X');
+    term.assertScreen("% e\u0301X");
+    term.assertAt(0, 4);
+
+    term.read(BACKWARD_KEY);
+    term.assertAt(0, 3);
+    term.read(BACKWARD_DELETE_KEY);
+    term.assertScreen("% eX");
+    term.assertAt(0, 3);
+
+    term.read('\r');
+    assertEquals("eX", line.get());
+  }
+
+  @Test
+  public void testSupplementaryUnicodeInput() {
+    TestTerm term = new TestTerm(this);
+    Supplier<String> line = term.readlineComplete();
+    int codePoint = 0x20000;
+    String value = new String(Character.toChars(codePoint));
+    term.read(codePoint, 'X');
+    term.assertScreen("% " + value + "X");
+    term.assertAt(0, 5);
+    term.read('\r');
+    assertEquals(value + "X", line.get());
+  }
+
   @Test
   public void testHistory() {
     TestTerm term = new TestTerm(this);