diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 0000000..4ed7d65 --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,39 @@ +name: Playwright + +on: + pull_request: + branches: + - main + +permissions: + contents: read + +jobs: + e2e: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 23 + + - name: Install dependencies + run: npm install + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Run Playwright tests + run: npm run test:e2e + + - name: Upload Playwright report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report/ + retention-days: 7 diff --git a/.gitignore b/.gitignore index f0d37f8..972fb92 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ lerna-debug.log* package-lock.json node_modules dist +playwright-report +test-results dist-ssr __pycache__/ *.py[cod] diff --git a/package.json b/package.json index 5b97e09..a13ebe8 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,8 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "test:e2e": "playwright test", + "test:e2e:ui": "playwright test --ui", "dev:website": "vite --config vite.config.website.ts", "build:website": "vite build --config vite.config.website.ts", "preview:website": "vite preview --config vite.config.website.ts", @@ -67,6 +69,7 @@ "@koa/router": "^15.4.0", "@multiavatar/multiavatar": "^1.0.7", "@pinia/testing": "^1.0.3", + "@playwright/test": "^1.60.0", "@types/eventsource": "^1.1.15", "@types/js-yaml": "^4.0.9", "@types/koa": "^2.15.0", @@ -116,4 +119,4 @@ "vue-tsc": "^3.2.8", "ws": "^8.20.0" } -} \ No newline at end of file +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..7afd801 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,33 @@ +/// +import { defineConfig, devices } from '@playwright/test' + +const PORT = Number(process.env.PLAYWRIGHT_PORT || 4173) +const BASE_URL = `http://127.0.0.1:${PORT}` + +export default defineConfig({ + testDir: './tests/e2e', + fullyParallel: true, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: process.env.CI ? [['dot'], ['html', { open: 'never' }]] : [['list']], + use: { + baseURL: BASE_URL, + locale: 'en-US', + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'retain-on-failure', + }, + webServer: { + command: `npx vite --host 127.0.0.1 --port ${PORT}`, + url: BASE_URL, + reuseExistingServer: !process.env.CI, + timeout: 120_000, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}) diff --git a/tests/e2e/auth.spec.ts b/tests/e2e/auth.spec.ts new file mode 100644 index 0000000..617f9eb --- /dev/null +++ b/tests/e2e/auth.spec.ts @@ -0,0 +1,44 @@ +import { expect, test } from '@playwright/test' +import { mockHermesApi, TEST_ACCESS_KEY } from './fixtures' + +test('redirects protected routes to the login screen without a token', async ({ page }) => { + const api = await mockHermesApi(page) + + await page.goto('/#/hermes/jobs') + + await expect(page).toHaveURL(/#\/$/) + await expect(page.getByRole('heading', { name: 'Hermes Web UI' })).toBeVisible() + await expect(page.getByPlaceholder('Access token')).toBeVisible() + expect(api.unexpectedRequests).toEqual([]) +}) + +test('rejects an invalid access token without persisting it', async ({ page }) => { + const api = await mockHermesApi(page, { tokenValidationStatus: 401 }) + + await page.goto('/') + await page.getByPlaceholder('Access token').fill('bad-token') + await page.getByRole('button', { name: 'Login' }).click() + + await expect(page.getByText('Invalid token')).toBeVisible() + await expect(page).toHaveURL(/#\/$/) + await expect(page.evaluate(() => window.localStorage.getItem('hermes_api_key'))).resolves.toBeNull() + expect(api.unexpectedRequests).toEqual([]) +}) + +test('validates token login through the BFF before entering the app', async ({ page }) => { + const api = await mockHermesApi(page) + + await page.goto('/') + await page.getByPlaceholder('Access token').fill(TEST_ACCESS_KEY) + await page.getByRole('button', { name: 'Login' }).click() + + await expect(page).toHaveURL(/#\/hermes\/chat$/) + await expect(page.evaluate(() => window.localStorage.getItem('hermes_api_key'))).resolves.toBe(TEST_ACCESS_KEY) + + const validationRequest = api.requests.find((request) => ( + request.pathname === '/api/hermes/sessions' && + request.headers.authorization === `Bearer ${TEST_ACCESS_KEY}` + )) + expect(validationRequest).toBeTruthy() + expect(api.unexpectedRequests).toEqual([]) +}) diff --git a/tests/e2e/authenticated-shell.spec.ts b/tests/e2e/authenticated-shell.spec.ts new file mode 100644 index 0000000..8ff1187 --- /dev/null +++ b/tests/e2e/authenticated-shell.spec.ts @@ -0,0 +1,26 @@ +import { expect, test } from '@playwright/test' +import { authenticate, mockHermesApi, TEST_ACCESS_KEY } from './fixtures' + +test('renders authenticated shell and navigates between key product routes', async ({ page }) => { + await authenticate(page, TEST_ACCESS_KEY, 'research') + const api = await mockHermesApi(page) + + await page.goto('/#/hermes/jobs') + + await expect(page.getByRole('heading', { name: 'Scheduled Jobs' })).toBeVisible() + await expect(page.getByText('Nightly Smoke')).toBeVisible() + + const jobsRequest = api.requests.find((request) => request.pathname === '/api/hermes/jobs') + expect(jobsRequest?.headers.authorization).toBe(`Bearer ${TEST_ACCESS_KEY}`) + expect(jobsRequest?.headers['x-hermes-profile']).toBe('research') + + await page.locator('aside.sidebar').getByRole('button', { name: /^Models$/ }).click() + await expect(page).toHaveURL(/#\/hermes\/models$/) + await expect(page.getByRole('heading', { name: 'Models' })).toBeVisible() + await expect(page.getByText('test-model').first()).toBeVisible() + + await page.locator('aside.sidebar').getByRole('button', { name: /^Settings$/ }).click() + await expect(page).toHaveURL(/#\/hermes\/settings$/) + await expect(page.getByRole('heading', { name: 'Settings' })).toBeVisible() + expect(api.unexpectedRequests).toEqual([]) +}) diff --git a/tests/e2e/fixtures.ts b/tests/e2e/fixtures.ts new file mode 100644 index 0000000..4ec776f --- /dev/null +++ b/tests/e2e/fixtures.ts @@ -0,0 +1,191 @@ +import type { Page, Request, Route } from '@playwright/test' + +export const TEST_ACCESS_KEY = 'playwright-access-key' + +export interface MockedRequest { + method: string + pathname: string + search: string + headers: Record + postData: string | null +} + +interface MockHermesApiOptions { + tokenValidationStatus?: number +} + +const sampleModelGroup = { + provider: 'test-provider', + label: 'Test Provider', + base_url: 'https://example.invalid/v1', + models: ['test-model'], + available_models: ['test-model'], + api_key: '', + builtin: true, +} + +const sampleJob = { + job_id: 'job-smoke', + id: 'job-smoke', + name: 'Nightly Smoke', + prompt: 'Run the smoke check', + prompt_preview: 'Run the smoke check', + skills: [], + skill: null, + model: 'test-model', + provider: 'test-provider', + base_url: null, + script: null, + schedule: '0 9 * * *', + schedule_display: '0 9 * * *', + repeat: { times: null, completed: 0 }, + enabled: true, + state: 'scheduled', + paused_at: null, + paused_reason: null, + created_at: '2026-01-01T00:00:00.000Z', + next_run_at: '2026-01-02T09:00:00.000Z', + last_run_at: null, + last_status: null, + last_error: null, + deliver: 'origin', + origin: null, + last_delivery_error: null, +} + +function jsonResponse(body: unknown, status = 200) { + return { + status, + contentType: 'application/json', + body: JSON.stringify(body), + } +} + +function recordRequest(request: Request): MockedRequest { + const url = new URL(request.url()) + return { + method: request.method(), + pathname: url.pathname, + search: url.search, + headers: request.headers(), + postData: request.postData(), + } +} + +export async function mockHermesApi(page: Page, options: MockHermesApiOptions = {}) { + const requests: MockedRequest[] = [] + const unexpectedRequests: MockedRequest[] = [] + const tokenValidationStatus = options.tokenValidationStatus ?? 200 + + await page.route('**/*', async (route: Route) => { + const request = route.request() + const url = new URL(request.url()) + const { pathname } = url + + if (!(pathname === '/health' || pathname.startsWith('/api/') || pathname.startsWith('/v1/'))) { + await route.continue() + return + } + + requests.push(recordRequest(request)) + + if (pathname === '/health') { + await route.fulfill(jsonResponse({ status: 'ok', webui_version: '0.5.23', node_version: '23.0.0' })) + return + } + + if (pathname === '/api/auth/status') { + await route.fulfill(jsonResponse({ hasPasswordLogin: false, username: null })) + return + } + + if (pathname === '/api/hermes/sessions') { + await route.fulfill(jsonResponse({ sessions: [] }, tokenValidationStatus)) + return + } + + if (pathname === '/api/hermes/sessions/hermes') { + await route.fulfill(jsonResponse({ sessions: [] })) + return + } + + if (pathname === '/api/hermes/sessions/context-length') { + await route.fulfill(jsonResponse({ context_length: 200000 })) + return + } + + if (pathname === '/api/hermes/files/list') { + await route.fulfill(jsonResponse({ entries: [], path: '' })) + return + } + + if (pathname === '/api/hermes/auth/copilot/check-token') { + await route.fulfill(jsonResponse({ has_token: false, source: null, enabled: false })) + return + } + + if (pathname === '/api/auth/locked-ips') { + await route.fulfill(jsonResponse({ locks: [] })) + return + } + + if (pathname === '/api/hermes/available-models') { + await route.fulfill(jsonResponse({ + default: 'test-model', + default_provider: 'test-provider', + groups: [sampleModelGroup], + allProviders: [sampleModelGroup], + model_aliases: {}, + model_visibility: {}, + })) + return + } + + if (pathname === '/api/hermes/profiles') { + await route.fulfill(jsonResponse({ + profiles: [ + { name: 'default', active: false, model: 'test-model', gateway: 'test', alias: 'Default' }, + { name: 'research', active: true, model: 'test-model', gateway: 'test', alias: 'Research' }, + ], + })) + return + } + + if (pathname === '/api/hermes/config') { + await route.fulfill(jsonResponse({ + display: { streaming: true, show_reasoning: true, show_cost: true }, + agent: {}, + memory: {}, + session_reset: {}, + privacy: {}, + approvals: {}, + })) + return + } + + if (pathname === '/api/hermes/jobs') { + await route.fulfill(jsonResponse({ jobs: [sampleJob] })) + return + } + + if (pathname === '/api/cron-history') { + await route.fulfill(jsonResponse({ runs: [] })) + return + } + + unexpectedRequests.push(recordRequest(request)) + await route.fulfill(jsonResponse({ error: `Unexpected mocked route: ${request.method()} ${pathname}` }, 404)) + }) + + return { requests, unexpectedRequests } +} + +export async function authenticate(page: Page, accessKey = TEST_ACCESS_KEY, profileName?: string) { + await page.addInitScript((state: { storedToken: string; storedProfileName?: string }) => { + const { storedToken, storedProfileName } = state + window.localStorage.setItem('hermes_api_key', storedToken) + if (storedProfileName) { + window.localStorage.setItem('hermes_active_profile_name', storedProfileName) + } + }, { storedToken: accessKey, storedProfileName: profileName }) +}