fix(android): carry the content description beside the label on the snapshot node (#2912)

An Android node is labelled by its text and falls back to the content description only when it has none, so an accessibility label the app set beside visible text never reached the wire: a labelled text view was named by its content, a filled field by its value, an empty field by its hint, a filled password field by its dots. The parser already read `content-desc`; the hierarchy node now carries it as `contentDescription` whenever it is not already the label, through the raw snapshot node, the unchanged-snapshot comparison, and the selector digest. `label` is unchanged, so nothing that keys on it moves.

Author: okwasniewski

packages/kernel/src/snapshot.ts

@@ -226,6 +226,13 @@ export type RawSnapshotNode = {
   subrole?: string;
   label?: string;
   value?: string;
+  /**
+   * Android content description when it is not already the `label`. An Android node is
+   * labelled by its text and falls back to the content description only when it has none,
+   * so an accessibility label the app set beside visible text (a labelled text view, a
+   * filled or hinted field) is carried here for consumers that want the accessible name.
+   */
+  contentDescription?: string;
   identifier?: string;
   rect?: Rect;
   enabled?: boolean;

packages/platform-android/src/ui-hierarchy-builder.ts

@@ -347,6 +347,7 @@ function createAndroidRawSnapshotNode(
     type: node.type ?? undefined,
     label: node.label ?? undefined,
     value: node.value ?? undefined,
+    contentDescription: node.contentDescription,
     identifier: node.identifier ?? undefined,
     bundleId: node.packageName ?? undefined,
     rect: node.rect,

packages/platform-android/src/ui-hierarchy-node.ts

@@ -8,6 +8,8 @@ export type AndroidUiHierarchy = {
   type: string | null;
   label: string | null;
   value: string | null;
+  /** The content description when the label is the text rather than it; see `RawSnapshotNode`. */
+  contentDescription?: string;
   identifier: string | null;
   packageName: string | null;
   rect?: Rect;

packages/platform-android/src/ui-hierarchy.ts

@@ -298,11 +298,13 @@ function normalizeAndroidUiHierarchyNode(
   depth: number,
   parent: AndroidNode,
 ): AndroidUiHierarchy {
+  const label = attrs.text || attrs.desc;
   return attachAndroidSiblingOrder(
     {
       type: attrs.className,
-      label: attrs.text || attrs.desc,
+      label,
       value: attrs.text,
+      ...(attrs.desc && attrs.desc !== label ? { contentDescription: attrs.desc } : {}),
       identifier: attrs.resourceId,
       packageName: attrs.packageName,
       rect: attrs.rect,

src/__tests__/android-ui-hierarchy.test.ts

@@ -41,6 +41,7 @@ test('parseUiHierarchy reads double-quoted Android node attributes', () => {
   assert.equal(result.nodes.length, 1);
   assert.equal(result.nodes[0]!.value, 'Hello');
   assert.equal(result.nodes[0]!.label, 'Hello');
+  assert.equal(result.nodes[0]!.contentDescription, 'Greeting');
   assert.equal(result.nodes[0]!.identifier, 'com.demo:id/title');
   assert.deepEqual(result.nodes[0]!.rect, { x: 10, y: 20, width: 100, height: 40 });
   assert.equal(result.nodes[0]!.hittable, true);
@@ -111,9 +112,30 @@ test('interactive Android snapshots keep Compose semantic views inside actionabl
 
   assert.equal(result.nodes[0]?.hittable, true);
   assert.equal(result.nodes[1]?.label, 'Start a call');
+  assert.equal(result.nodes[1]?.contentDescription, undefined);
   assert.equal(result.nodes[1]?.parentIndex, 0);
 });
 
+test('an Android content description beside text is carried, and one that is the label is not repeated', () => {
+  const xml = `<hierarchy>
+  <node class="android.widget.EditText" package="com.example.app" text="Ada Lovelace" content-desc="Name field" resource-id="name-input" bounds="[0,0][390,44]" visible-to-user="true"/>
+  <node class="android.widget.EditText" package="com.example.app" text="Type your name" content-desc="Name field" hint-showing="true" resource-id="empty-input" bounds="[0,60][390,104]" visible-to-user="true"/>
+  <node class="android.widget.TextView" package="com.example.app" text="Plain controls" content-desc="Plain controls" bounds="[0,120][390,164]" visible-to-user="true"/>
+  <node class="android.widget.TextView" package="com.example.app" text="Untitled" bounds="[0,180][390,224]" visible-to-user="true"/>
+</hierarchy>`;
+
+  const result = parseUiHierarchy(xml, 800, { raw: true });
+
+  assert.equal(result.nodes[0]?.label, 'Ada Lovelace');
+  assert.equal(result.nodes[0]?.contentDescription, 'Name field');
+  assert.equal(result.nodes[1]?.label, 'Type your name');
+  assert.equal(result.nodes[1]?.hintShowing, true);
+  assert.equal(result.nodes[1]?.contentDescription, 'Name field');
+  assert.equal(result.nodes[2]?.label, 'Plain controls');
+  assert.equal(result.nodes[2]?.contentDescription, undefined);
+  assert.equal(result.nodes[3]?.contentDescription, undefined);
+});
+
 test('interactive Android snapshots keep a fixed sibling outside filtered scroll content (#1377)', () => {
   const xml = `<hierarchy>
   <node class="android.widget.FrameLayout" bounds="[0,0][400,800]" visible-to-user="true">

src/commands/capture/runtime/snapshot-unchanged.ts

@@ -107,6 +107,7 @@ const PRESENTATION_SCALAR_FIELDS = {
   subrole: true,
   label: true,
   value: true,
+  contentDescription: true,
   identifier: true,
   enabled: true,
   selected: true,

src/daemon/response-views.ts

@@ -110,6 +110,7 @@ const SELECTOR_DIGEST_NODE_FIELDS = [
   'subrole',
   'label',
   'value',
+  'contentDescription',
   'identifier',
   'enabled',
   'selected',