From 97b15d6514674745e5f4e974e1a18a0bc39996ec Mon Sep 17 00:00:00 2001
From: Zhicheng Han <43314240+hanzckernel@users.noreply.github.com>
Date: Fri, 15 May 2026 11:35:10 +0200
Subject: [PATCH] =?UTF-8?q?test:=20=E6=B7=BB=E5=8A=A0=E6=B5=8F=E8=A7=88?=
=?UTF-8?q?=E5=99=A8=E7=83=9F=E6=B5=8B=E5=A5=97=E4=BB=B6=20(#750)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
* test: add Playwright browser smoke suite
* Update playwright.yml
---------
Co-authored-by: ekko <152005280+EKKOLearnAI@users.noreply.github.com>
---
.github/workflows/playwright.yml | 39 ++++++
.gitignore | 2 +
package.json | 5 +-
playwright.config.ts | 33 +++++
tests/e2e/auth.spec.ts | 44 ++++++
tests/e2e/authenticated-shell.spec.ts | 26 ++++
tests/e2e/fixtures.ts | 191 ++++++++++++++++++++++++++
7 files changed, 339 insertions(+), 1 deletion(-)
create mode 100644 .github/workflows/playwright.yml
create mode 100644 playwright.config.ts
create mode 100644 tests/e2e/auth.spec.ts
create mode 100644 tests/e2e/authenticated-shell.spec.ts
create mode 100644 tests/e2e/fixtures.ts
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
new file mode 100644
index 0000000..4ed7d65
--- /dev/null
+++ b/.github/workflows/playwright.yml
@@ -0,0 +1,39 @@
+name: Playwright
+
+on:
+ pull_request:
+ branches:
+ - main
+
+permissions:
+ contents: read
+
+jobs:
+ e2e:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup Node.js
+ uses: actions/setup-node@v4
+ with:
+ node-version: 23
+
+ - name: Install dependencies
+ run: npm install
+
+ - name: Install Playwright browsers
+ run: npx playwright install --with-deps chromium
+
+ - name: Run Playwright tests
+ run: npm run test:e2e
+
+ - name: Upload Playwright report
+ if: ${{ !cancelled() }}
+ uses: actions/upload-artifact@v4
+ with:
+ name: playwright-report
+ path: playwright-report/
+ retention-days: 7
diff --git a/.gitignore b/.gitignore
index f0d37f8..972fb92 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,8 @@ lerna-debug.log*
package-lock.json
node_modules
dist
+playwright-report
+test-results
dist-ssr
__pycache__/
*.py[cod]
diff --git a/package.json b/package.json
index 5b97e09..a13ebe8 100644
--- a/package.json
+++ b/package.json
@@ -43,6 +43,8 @@
"test": "vitest run",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage",
+ "test:e2e": "playwright test",
+ "test:e2e:ui": "playwright test --ui",
"dev:website": "vite --config vite.config.website.ts",
"build:website": "vite build --config vite.config.website.ts",
"preview:website": "vite preview --config vite.config.website.ts",
@@ -67,6 +69,7 @@
"@koa/router": "^15.4.0",
"@multiavatar/multiavatar": "^1.0.7",
"@pinia/testing": "^1.0.3",
+ "@playwright/test": "^1.60.0",
"@types/eventsource": "^1.1.15",
"@types/js-yaml": "^4.0.9",
"@types/koa": "^2.15.0",
@@ -116,4 +119,4 @@
"vue-tsc": "^3.2.8",
"ws": "^8.20.0"
}
-}
\ No newline at end of file
+}
diff --git a/playwright.config.ts b/playwright.config.ts
new file mode 100644
index 0000000..7afd801
--- /dev/null
+++ b/playwright.config.ts
@@ -0,0 +1,33 @@
+///
+import { defineConfig, devices } from '@playwright/test'
+
+const PORT = Number(process.env.PLAYWRIGHT_PORT || 4173)
+const BASE_URL = `http://127.0.0.1:${PORT}`
+
+export default defineConfig({
+ testDir: './tests/e2e',
+ fullyParallel: true,
+ forbidOnly: !!process.env.CI,
+ retries: process.env.CI ? 2 : 0,
+ workers: process.env.CI ? 1 : undefined,
+ reporter: process.env.CI ? [['dot'], ['html', { open: 'never' }]] : [['list']],
+ use: {
+ baseURL: BASE_URL,
+ locale: 'en-US',
+ trace: 'retain-on-failure',
+ screenshot: 'only-on-failure',
+ video: 'retain-on-failure',
+ },
+ webServer: {
+ command: `npx vite --host 127.0.0.1 --port ${PORT}`,
+ url: BASE_URL,
+ reuseExistingServer: !process.env.CI,
+ timeout: 120_000,
+ },
+ projects: [
+ {
+ name: 'chromium',
+ use: { ...devices['Desktop Chrome'] },
+ },
+ ],
+})
diff --git a/tests/e2e/auth.spec.ts b/tests/e2e/auth.spec.ts
new file mode 100644
index 0000000..617f9eb
--- /dev/null
+++ b/tests/e2e/auth.spec.ts
@@ -0,0 +1,44 @@
+import { expect, test } from '@playwright/test'
+import { mockHermesApi, TEST_ACCESS_KEY } from './fixtures'
+
+test('redirects protected routes to the login screen without a token', async ({ page }) => {
+ const api = await mockHermesApi(page)
+
+ await page.goto('/#/hermes/jobs')
+
+ await expect(page).toHaveURL(/#\/$/)
+ await expect(page.getByRole('heading', { name: 'Hermes Web UI' })).toBeVisible()
+ await expect(page.getByPlaceholder('Access token')).toBeVisible()
+ expect(api.unexpectedRequests).toEqual([])
+})
+
+test('rejects an invalid access token without persisting it', async ({ page }) => {
+ const api = await mockHermesApi(page, { tokenValidationStatus: 401 })
+
+ await page.goto('/')
+ await page.getByPlaceholder('Access token').fill('bad-token')
+ await page.getByRole('button', { name: 'Login' }).click()
+
+ await expect(page.getByText('Invalid token')).toBeVisible()
+ await expect(page).toHaveURL(/#\/$/)
+ await expect(page.evaluate(() => window.localStorage.getItem('hermes_api_key'))).resolves.toBeNull()
+ expect(api.unexpectedRequests).toEqual([])
+})
+
+test('validates token login through the BFF before entering the app', async ({ page }) => {
+ const api = await mockHermesApi(page)
+
+ await page.goto('/')
+ await page.getByPlaceholder('Access token').fill(TEST_ACCESS_KEY)
+ await page.getByRole('button', { name: 'Login' }).click()
+
+ await expect(page).toHaveURL(/#\/hermes\/chat$/)
+ await expect(page.evaluate(() => window.localStorage.getItem('hermes_api_key'))).resolves.toBe(TEST_ACCESS_KEY)
+
+ const validationRequest = api.requests.find((request) => (
+ request.pathname === '/api/hermes/sessions' &&
+ request.headers.authorization === `Bearer ${TEST_ACCESS_KEY}`
+ ))
+ expect(validationRequest).toBeTruthy()
+ expect(api.unexpectedRequests).toEqual([])
+})
diff --git a/tests/e2e/authenticated-shell.spec.ts b/tests/e2e/authenticated-shell.spec.ts
new file mode 100644
index 0000000..8ff1187
--- /dev/null
+++ b/tests/e2e/authenticated-shell.spec.ts
@@ -0,0 +1,26 @@
+import { expect, test } from '@playwright/test'
+import { authenticate, mockHermesApi, TEST_ACCESS_KEY } from './fixtures'
+
+test('renders authenticated shell and navigates between key product routes', async ({ page }) => {
+ await authenticate(page, TEST_ACCESS_KEY, 'research')
+ const api = await mockHermesApi(page)
+
+ await page.goto('/#/hermes/jobs')
+
+ await expect(page.getByRole('heading', { name: 'Scheduled Jobs' })).toBeVisible()
+ await expect(page.getByText('Nightly Smoke')).toBeVisible()
+
+ const jobsRequest = api.requests.find((request) => request.pathname === '/api/hermes/jobs')
+ expect(jobsRequest?.headers.authorization).toBe(`Bearer ${TEST_ACCESS_KEY}`)
+ expect(jobsRequest?.headers['x-hermes-profile']).toBe('research')
+
+ await page.locator('aside.sidebar').getByRole('button', { name: /^Models$/ }).click()
+ await expect(page).toHaveURL(/#\/hermes\/models$/)
+ await expect(page.getByRole('heading', { name: 'Models' })).toBeVisible()
+ await expect(page.getByText('test-model').first()).toBeVisible()
+
+ await page.locator('aside.sidebar').getByRole('button', { name: /^Settings$/ }).click()
+ await expect(page).toHaveURL(/#\/hermes\/settings$/)
+ await expect(page.getByRole('heading', { name: 'Settings' })).toBeVisible()
+ expect(api.unexpectedRequests).toEqual([])
+})
diff --git a/tests/e2e/fixtures.ts b/tests/e2e/fixtures.ts
new file mode 100644
index 0000000..4ec776f
--- /dev/null
+++ b/tests/e2e/fixtures.ts
@@ -0,0 +1,191 @@
+import type { Page, Request, Route } from '@playwright/test'
+
+export const TEST_ACCESS_KEY = 'playwright-access-key'
+
+export interface MockedRequest {
+ method: string
+ pathname: string
+ search: string
+ headers: Record
+ postData: string | null
+}
+
+interface MockHermesApiOptions {
+ tokenValidationStatus?: number
+}
+
+const sampleModelGroup = {
+ provider: 'test-provider',
+ label: 'Test Provider',
+ base_url: 'https://example.invalid/v1',
+ models: ['test-model'],
+ available_models: ['test-model'],
+ api_key: '',
+ builtin: true,
+}
+
+const sampleJob = {
+ job_id: 'job-smoke',
+ id: 'job-smoke',
+ name: 'Nightly Smoke',
+ prompt: 'Run the smoke check',
+ prompt_preview: 'Run the smoke check',
+ skills: [],
+ skill: null,
+ model: 'test-model',
+ provider: 'test-provider',
+ base_url: null,
+ script: null,
+ schedule: '0 9 * * *',
+ schedule_display: '0 9 * * *',
+ repeat: { times: null, completed: 0 },
+ enabled: true,
+ state: 'scheduled',
+ paused_at: null,
+ paused_reason: null,
+ created_at: '2026-01-01T00:00:00.000Z',
+ next_run_at: '2026-01-02T09:00:00.000Z',
+ last_run_at: null,
+ last_status: null,
+ last_error: null,
+ deliver: 'origin',
+ origin: null,
+ last_delivery_error: null,
+}
+
+function jsonResponse(body: unknown, status = 200) {
+ return {
+ status,
+ contentType: 'application/json',
+ body: JSON.stringify(body),
+ }
+}
+
+function recordRequest(request: Request): MockedRequest {
+ const url = new URL(request.url())
+ return {
+ method: request.method(),
+ pathname: url.pathname,
+ search: url.search,
+ headers: request.headers(),
+ postData: request.postData(),
+ }
+}
+
+export async function mockHermesApi(page: Page, options: MockHermesApiOptions = {}) {
+ const requests: MockedRequest[] = []
+ const unexpectedRequests: MockedRequest[] = []
+ const tokenValidationStatus = options.tokenValidationStatus ?? 200
+
+ await page.route('**/*', async (route: Route) => {
+ const request = route.request()
+ const url = new URL(request.url())
+ const { pathname } = url
+
+ if (!(pathname === '/health' || pathname.startsWith('/api/') || pathname.startsWith('/v1/'))) {
+ await route.continue()
+ return
+ }
+
+ requests.push(recordRequest(request))
+
+ if (pathname === '/health') {
+ await route.fulfill(jsonResponse({ status: 'ok', webui_version: '0.5.23', node_version: '23.0.0' }))
+ return
+ }
+
+ if (pathname === '/api/auth/status') {
+ await route.fulfill(jsonResponse({ hasPasswordLogin: false, username: null }))
+ return
+ }
+
+ if (pathname === '/api/hermes/sessions') {
+ await route.fulfill(jsonResponse({ sessions: [] }, tokenValidationStatus))
+ return
+ }
+
+ if (pathname === '/api/hermes/sessions/hermes') {
+ await route.fulfill(jsonResponse({ sessions: [] }))
+ return
+ }
+
+ if (pathname === '/api/hermes/sessions/context-length') {
+ await route.fulfill(jsonResponse({ context_length: 200000 }))
+ return
+ }
+
+ if (pathname === '/api/hermes/files/list') {
+ await route.fulfill(jsonResponse({ entries: [], path: '' }))
+ return
+ }
+
+ if (pathname === '/api/hermes/auth/copilot/check-token') {
+ await route.fulfill(jsonResponse({ has_token: false, source: null, enabled: false }))
+ return
+ }
+
+ if (pathname === '/api/auth/locked-ips') {
+ await route.fulfill(jsonResponse({ locks: [] }))
+ return
+ }
+
+ if (pathname === '/api/hermes/available-models') {
+ await route.fulfill(jsonResponse({
+ default: 'test-model',
+ default_provider: 'test-provider',
+ groups: [sampleModelGroup],
+ allProviders: [sampleModelGroup],
+ model_aliases: {},
+ model_visibility: {},
+ }))
+ return
+ }
+
+ if (pathname === '/api/hermes/profiles') {
+ await route.fulfill(jsonResponse({
+ profiles: [
+ { name: 'default', active: false, model: 'test-model', gateway: 'test', alias: 'Default' },
+ { name: 'research', active: true, model: 'test-model', gateway: 'test', alias: 'Research' },
+ ],
+ }))
+ return
+ }
+
+ if (pathname === '/api/hermes/config') {
+ await route.fulfill(jsonResponse({
+ display: { streaming: true, show_reasoning: true, show_cost: true },
+ agent: {},
+ memory: {},
+ session_reset: {},
+ privacy: {},
+ approvals: {},
+ }))
+ return
+ }
+
+ if (pathname === '/api/hermes/jobs') {
+ await route.fulfill(jsonResponse({ jobs: [sampleJob] }))
+ return
+ }
+
+ if (pathname === '/api/cron-history') {
+ await route.fulfill(jsonResponse({ runs: [] }))
+ return
+ }
+
+ unexpectedRequests.push(recordRequest(request))
+ await route.fulfill(jsonResponse({ error: `Unexpected mocked route: ${request.method()} ${pathname}` }, 404))
+ })
+
+ return { requests, unexpectedRequests }
+}
+
+export async function authenticate(page: Page, accessKey = TEST_ACCESS_KEY, profileName?: string) {
+ await page.addInitScript((state: { storedToken: string; storedProfileName?: string }) => {
+ const { storedToken, storedProfileName } = state
+ window.localStorage.setItem('hermes_api_key', storedToken)
+ if (storedProfileName) {
+ window.localStorage.setItem('hermes_active_profile_name', storedProfileName)
+ }
+ }, { storedToken: accessKey, storedProfileName: profileName })
+}