Skip to content

Commit 9873233

Browse files
testlabautotestlabauto
authored
E2E test: create pyrefly suite infrstructure (#10890)
Just getting a workflow in place to iterate on pyrefly tests ### QA Notes
1 parent d46ce10 commit 9873233

File tree

8 files changed

+383
-203
lines changed

8 files changed

+383
-203
lines changed

‎.github/workflows/test-e2e-pyrefly.yml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: "Test: E2E Pyrefly (Ubuntu)"
2+
3+
on:
4+
workflow_dispatch:
5+
6+
permissions:
7+
id-token: write
8+
contents: read
9+
packages: read
10+
11+
jobs:
12+
e2e-linux:
13+
name: 'e2e-pyrefly-ubuntu'
14+
timeout-minutes: 60
15+
runs-on: ubuntu-latest-8x
16+
container:
17+
image: ghcr.io/posit-dev/positron-ubuntu24-amd64:64
18+
options: --user 0:0
19+
# Static PAT is needed because the bot can't pass a token to the job for security reasons
20+
credentials:
21+
username: ${{ secrets.POSITRON_GITHUB_RO_USER }}
22+
password: ${{ secrets.POSITRON_GITHUB_RO_PAT }}
23+
24+
env:
25+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
26+
POSITRON_BUILD_NUMBER: 0 # CI skips building releases
27+
_R_CHECK_FUTURE_FILE_TIMESTAMPS_: false # this check can be flaky in the R pkg tests
28+
_R_CHECK_CRAN_INCOMING_: false
29+
_R_CHECK_SYSTEM_CLOCK_: false
30+
AWS_S3_BUCKET: positron-test-reports
31+
R_LIBS_SITE: /usr/local/lib/R/site-library
32+
R_LIBS_USER: /usr/local/lib/R/site-library
33+
34+
steps:
35+
- uses: actions/checkout@v5
36+
with:
37+
fetch-depth: 0
38+
submodules: recursive
39+
40+
- name: Setup Build and Compile
41+
uses: ./.github/actions/setup-build-env
42+
continue-on-error: true
43+
44+
- name: Setup E2E Test Environment
45+
uses: ./.github/actions/setup-test-env
46+
with:
47+
aws-role-to-assume: ${{ secrets.QA_AWS_RO_ROLE }}
48+
aws-region: ${{ secrets.QA_AWS_REGION }}
49+
50+
- name: Send Results to GH Summary
51+
uses: ./.github/actions/gen-report-dir
52+
53+
- name: Alter AppArmor Restrictions for Playwright
54+
run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
55+
56+
- name: Run Playwright Tests
57+
shell: bash
58+
env:
59+
POSITRON_PY_VER_SEL: "3.10.12"
60+
POSITRON_R_VER_SEL: 4.4.0
61+
POSITRON_PY_ALT_VER_SEL: "3.13.0"
62+
POSITRON_R_ALT_VER_SEL: 4.4.2
63+
POSITRON_HIDDEN_PY: "3.12.10 (Conda)"
64+
POSITRON_HIDDEN_R: 4.4.1
65+
COMMIT_INFO_MESSAGE: ${{ github.event.head_commit.message }}
66+
PWTEST_BLOB_DO_NOT_REMOVE: 1
67+
USE_KEY: true
68+
GH_SUMMARY_REPORT: true
69+
run: |
70+
SKIP_BOOTSTRAP=true npx playwright test --project e2e-elctron test/e2e/tests/pyrefly --workers 1 --reporter=json
71+
72+
- name: Upload Playwright Report to S3
73+
if: ${{ success() || failure() }}
74+
uses: ./.github/actions/upload-report-to-s3
75+
with:
76+
role-to-assume: ${{ secrets.AWS_TEST_REPORTS_ROLE }}
77+
report-dir: ${{ env.REPORT_DIR }}

‎playwright.config.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ export default defineConfig<ExtendedTestOptions>({
3131
'example.test.ts',
3232
'**/workbench/**',
3333
'**/inspect-ai/**',
34-
'**/remote-ssh/**'
34+
'**/remote-ssh/**',
35+
'**/pyrefly/**'
3536
],
3637
fullyParallel: false, // Run individual tests w/in a spec in parallel
3738
forbidOnly: !!process.env.CI,
@@ -150,6 +151,7 @@ export default defineConfig<ExtendedTestOptions>({
150151
testIgnore: [
151152
'example.test.ts',
152153
'**/workbench/**',
154+
'**/remote-ssh/**'
153155
],
154156
use: {
155157
artifactDir: 'inspect-ai',
@@ -160,7 +162,8 @@ export default defineConfig<ExtendedTestOptions>({
160162
name: 'e2e-workbench',
161163
testIgnore: [
162164
'example.test.ts',
163-
'**/inspect-ai/**'
165+
'**/inspect-ai/**',
166+
'**/remote-ssh/**'
164167
],
165168
use: {
166169
artifactDir: 'e2e-workbench',
@@ -175,7 +178,8 @@ export default defineConfig<ExtendedTestOptions>({
175178
name: 'e2e-remote-ssh',
176179
testIgnore: [
177180
'example.test.ts',
178-
'**/inspect-ai/**'
181+
'**/inspect-ai/**',
182+
'**/workbench/**',
179183
],
180184
use: {
181185
artifactDir: 'e2e-remote-ssh',

‎test/e2e/tests/diagnostics/diagnostics.test.ts‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -17,45 +17,6 @@ test.describe('Diagnostics', {
1717
await runCommand('workbench.action.closeAllEditors');
1818
});
1919

20-
test.skip('Python - Verify diagnostics isolation between sessions in the editor and problems view', async function ({ app, runCommand, sessions }) {
21-
const { problems, editor, console } = app.workbench;
22-
23-
// Start Python Session and install 'termcolor'
24-
const pySession = await sessions.start('python');
25-
await console.executeCode('Python', 'pip install termcolor', { maximizeConsole: false });
26-
27-
// Open new Python file and use 'termcolor'
28-
await runCommand('Python: New File');
29-
await editor.type('import termcolor\n\ntermcolor.COLORS.copy()\n');
30-
31-
// Python Session 1 - verify no problems
32-
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
33-
await problems.expectSquigglyCountToBe('warning', 0);
34-
35-
// Python Session 1 - restart session and verify no problems
36-
await sessions.restart(pySession.id);
37-
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
38-
await problems.expectSquigglyCountToBe('warning', 0);
39-
40-
// Start Python Session 2 (same runtime) - verify no problems
41-
const pySession2 = await sessions.start('python', { reuse: false });
42-
await sessions.select(pySession2.id);
43-
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
44-
await problems.expectSquigglyCountToBe('warning', 0);
45-
46-
// Python Alt Session - verify warning since pkg not installed
47-
await sessions.start('pythonAlt');
48-
await problems.expectDiagnosticsToBe({ badgeCount: 1, warningCount: 1, errorCount: 0 });
49-
await problems.expectWarningText('Import "termcolor" could not be resolved');
50-
await problems.expectSquigglyCountToBe('warning', 1);
51-
52-
// Python Session 1 - restart session and verify no problems
53-
await sessions.select(pySession.id);
54-
await sessions.restart(pySession.id);
55-
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
56-
await problems.expectSquigglyCountToBe('warning', 0);
57-
});
58-
5920
test('R - Verify diagnostics isolation between sessions in the editor and problems view', {
6021
tag: [tags.ARK]
6122
}, async function ({ app, runCommand, sessions }) {

‎test/e2e/tests/outline/outline.test.ts‎

Lines changed: 0 additions & 144 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,8 @@
44
*--------------------------------------------------------------------------------------------*/
55

66
import { join } from 'path';
7-
import { Outline } from '../../pages/outline.js';
87
import { test, tags } from '../_test.setup.js';
98

10-
const R_FILE = 'basic-outline-with-vars.r';
11-
const PY_FILE = 'basic-outline-with-vars.py';
129

1310
test.use({
1411
suiteId: __filename
@@ -20,138 +17,8 @@ test.describe('Outline', { tag: [tags.WEB, tags.WIN, tags.OUTLINE] }, () => {
2017
await hotKeys.closeAllEditors();
2118
});
2219

23-
test.describe('Outline: Sessions', { tag: [tags.SESSIONS, tags.ARK] }, () => {
24-
25-
test.beforeAll(async function ({ app, openFile, hotKeys }) {
26-
const { outline } = app.workbench;
27-
28-
await openFile(`workspaces/outline/${PY_FILE}`);
29-
await openFile(`workspaces/outline/${R_FILE}`);
30-
31-
await hotKeys.closeSecondarySidebar();
32-
await outline.focus();
33-
});
34-
35-
test.skip('Verify outline is based on editor and per session', async function ({ app, sessions }) {
36-
const { outline, console, editor } = app.workbench;
37-
38-
// No active session - verify no outlines
39-
await editor.selectTab(PY_FILE);
40-
await outline.expectOutlineToBeEmpty();
41-
await editor.selectTab(R_FILE);
42-
await outline.expectOutlineToBeEmpty();
43-
44-
// Start sessions
45-
const [pySession1, pySession2, rSession1, rSession2] = await sessions.start(['python', 'pythonAlt', 'r', 'rAlt']);
46-
47-
// Select Python file
48-
await editor.selectTab(PY_FILE);
49-
await verifyPythonOutline(outline);
50-
51-
// Select R Session 1 - verify Python outline
52-
// Use last-active Python session's LSP for Python files, even if foreground session is R.
53-
await sessions.select(rSession1.id);
54-
await verifyPythonOutline(outline);
55-
56-
// Select Python Session 1 - verify Python outline
57-
await sessions.select(pySession1.id);
58-
await console.typeToConsole('global_variable="goodbye"', true);
59-
await verifyPythonOutline(outline);
60-
61-
// Select R file
62-
await editor.selectTab(R_FILE);
63-
await verifyROutline(outline);
64-
65-
// Select R Session 1 - verify R outline
66-
await sessions.select(rSession1.id);
67-
await verifyROutline(outline);
68-
69-
// Select R Session 2 - verify R outline
70-
await sessions.select(rSession2.id);
71-
await verifyROutline(outline);
72-
73-
// Select Python file - verify Python outline
74-
await editor.selectTab(PY_FILE);
75-
await verifyPythonOutline(outline);
76-
77-
// Python Session 2 - verify Python outline
78-
await sessions.select(pySession2.id);
79-
await console.typeToConsole('global_variable="goodbye2"', true);
80-
await verifyPythonOutline(outline);
81-
});
82-
83-
test.skip('Verify outline after reload with Python in foreground and R in background', {
84-
annotation: [{ type: 'issue', description: 'https://github.com/posit-dev/positron/issues/7052' }],
85-
}, async function ({ app, runCommand, sessions }) {
86-
const { outline, editor } = app.workbench;
87-
88-
// Start sessions
89-
await sessions.deleteAll();
90-
const [, rSession] = await sessions.start(['python', 'r']);
91-
92-
// Verify outlines for both file types
93-
await editor.selectTab(PY_FILE);
94-
await verifyPythonOutline(outline);
95-
96-
await editor.selectTab(R_FILE);
97-
await verifyROutline(outline);
98-
99-
// Reload window
100-
await sessions.expectSessionCountToBe(2);
101-
await runCommand('workbench.action.reloadWindow');
102-
await sessions.expectSessionCountToBe(2);
103-
104-
// Verify outlines for both file types
105-
await editor.selectTab(PY_FILE);
106-
await verifyPythonOutline(outline);
107-
108-
await editor.selectTab(R_FILE);
109-
await sessions.select(rSession.id); // Issue 7052 - we shouldn't have to click the tab
110-
await verifyROutline(outline);
111-
});
112-
113-
test.skip('Verify outline after reload with R in foreground and Python in background', {
114-
annotation: [{ type: 'issue', description: 'https://github.com/posit-dev/positron/issues/7052' }],
115-
},
116-
async function ({ app, runCommand, sessions }) {
117-
const { outline, editor } = app.workbench;
118-
119-
// Start sessions
120-
await sessions.deleteAll();
121-
await sessions.start(['r', 'python']);
122-
123-
// Verify outlines for both file types
124-
await editor.selectTab(R_FILE);
125-
await verifyROutline(outline);
126-
127-
await editor.selectTab(PY_FILE);
128-
await verifyPythonOutline(outline);
129-
130-
// Reload window
131-
await runCommand('workbench.action.reloadWindow');
132-
133-
// Verify outlines for both file types
134-
await editor.selectTab(R_FILE);
135-
await verifyROutline(outline);
136-
137-
await editor.selectTab(PY_FILE);
138-
await verifyPythonOutline(outline);
139-
});
140-
});
141-
14220
test.describe('Outline: Basic', () => {
14321

144-
test.skip('Python - Verify Outline Contents', async function ({ app, python, openFile }) {
145-
await openFile(join('workspaces', 'chinook-db-py', 'chinook-sqlite.py'));
146-
await app.workbench.outline.expectOutlineToContain([
147-
'data_file_path',
148-
'conn',
149-
'cur',
150-
'rows',
151-
'df'
152-
]);
153-
});
154-
15522
test('R - Verify Outline Contents', {
15623
tag: [tags.ARK]
15724
}, async function ({ app, r, openFile }) {
@@ -166,14 +33,3 @@ test.describe('Outline', { tag: [tags.WEB, tags.WIN, tags.OUTLINE] }, () => {
16633

16734
});
16835

169-
async function verifyPythonOutline(outline: Outline) {
170-
await outline.expectOutlineElementCountToBe(2); // ensure no dupes from multisessions
171-
await outline.expectOutlineElementToBeVisible('global_variable = "hello"');
172-
await outline.expectOutlineElementToBeVisible('def demonstrate_scope');
173-
}
174-
175-
async function verifyROutline(outline: Outline) {
176-
await outline.expectOutlineElementCountToBe(2); // ensure no dupes from multisessions
177-
await outline.expectOutlineElementToBeVisible('demonstrate_scope');
178-
await outline.expectOutlineElementToBeVisible('global_variable');
179-
}

‎test/e2e/tests/pyrefly/pyrefly-diagnostics.test.ts‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (C) 2025 Posit Software, PBC. All rights reserved.
3+
* Licensed under the Elastic License 2.0. See LICENSE.txt for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { test, tags } from '../_test.setup.js';
7+
8+
test.use({
9+
suiteId: __filename
10+
});
11+
12+
test.describe('Diagnostics', {
13+
tag: [tags.SESSIONS, tags.PROBLEMS, tags.WEB, tags.WIN, tags.SOFT_FAIL],
14+
}, () => {
15+
16+
test.afterEach(async function ({ runCommand }) {
17+
await runCommand('workbench.action.closeAllEditors');
18+
});
19+
20+
test('Python - Verify diagnostics isolation between sessions in the editor and problems view', async function ({ app, runCommand, sessions }) {
21+
const { problems, editor, console } = app.workbench;
22+
23+
// Start Python Session and install 'termcolor'
24+
const pySession = await sessions.start('python');
25+
await console.executeCode('Python', 'pip install termcolor', { maximizeConsole: false });
26+
27+
// Open new Python file and use 'termcolor'
28+
await runCommand('Python: New File');
29+
await editor.type('import termcolor\n\ntermcolor.COLORS.copy()\n');
30+
31+
// Python Session 1 - verify no problems
32+
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
33+
await problems.expectSquigglyCountToBe('warning', 0);
34+
35+
// Python Session 1 - restart session and verify no problems
36+
await sessions.restart(pySession.id);
37+
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
38+
await problems.expectSquigglyCountToBe('warning', 0);
39+
40+
// Start Python Session 2 (same runtime) - verify no problems
41+
const pySession2 = await sessions.start('python', { reuse: false });
42+
await sessions.select(pySession2.id);
43+
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
44+
await problems.expectSquigglyCountToBe('warning', 0);
45+
46+
// Python Alt Session - verify warning since pkg not installed
47+
await sessions.start('pythonAlt');
48+
await problems.expectDiagnosticsToBe({ badgeCount: 1, warningCount: 1, errorCount: 0 });
49+
await problems.expectWarningText('Import "termcolor" could not be resolved');
50+
await problems.expectSquigglyCountToBe('warning', 1);
51+
52+
// Python Session 1 - restart session and verify no problems
53+
await sessions.select(pySession.id);
54+
await sessions.restart(pySession.id);
55+
await problems.expectDiagnosticsToBe({ badgeCount: 0, warningCount: 0, errorCount: 0 });
56+
await problems.expectSquigglyCountToBe('warning', 0);
57+
});
58+
59+
});

0 commit comments

Comments
 (0)