admin/Hermes-ui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: align usage analytics with Hermes state db (#350)

Browse Source
This commit is contained in:
Zhicheng Han
2026-04-30 13:46:31 +02:00
committed by GitHub
parent 05f15da90b
commit dac9006b3e
10 changed files with 591 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/server/sessions-controller.test.ts
+70
View File
@@ -5,8 +5,11 @@ const getConversationDetailFromDbMock = vi.fn()
const listConversationSummariesMock = vi.fn()
const getConversationDetailMock = vi.fn()
const getSessionDetailFromDbMock = vi.fn()
const getUsageStatsFromDbMock = vi.fn()
const getSessionMock = vi.fn()
const getGroupChatServerMock = vi.fn()
const getLocalUsageStatsMock = vi.fn()
const getActiveProfileNameMock = vi.fn()
const loggerWarnMock = vi.fn()
vi.mock('../../packages/server/src/db/hermes/conversations-db', () => ({
@@ -37,6 +40,7 @@ vi.mock('../../packages/server/src/db/hermes/sessions-db', () => ({
listSessionSummaries: vi.fn(),
searchSessionSummaries: vi.fn(),
getSessionDetailFromDb: getSessionDetailFromDbMock,
getUsageStatsFromDb: getUsageStatsFromDbMock,
}))
// Mock useLocalSessionStore to return false so we test the CLI path
@@ -48,6 +52,7 @@ vi.mock('../../packages/server/src/db/hermes/usage-store', () => ({
deleteUsage: vi.fn(),
getUsage: vi.fn(),
getUsageBatch: vi.fn(),
getLocalUsageStats: getLocalUsageStatsMock,
}))
vi.mock('../../packages/server/src/routes/hermes/group-chat', () => ({
@@ -58,6 +63,10 @@ vi.mock('../../packages/server/src/services/hermes/model-context', () => ({
getModelContextLength: vi.fn(),
}))
vi.mock('../../packages/server/src/services/hermes/hermes-profile', () => ({
getActiveProfileName: getActiveProfileNameMock,
}))
describe('session conversations controller', () => {
beforeEach(() => {
vi.resetModules()
@@ -66,9 +75,13 @@ describe('session conversations controller', () => {
listConversationSummariesMock.mockReset()
getConversationDetailMock.mockReset()
getSessionDetailFromDbMock.mockReset()
getUsageStatsFromDbMock.mockReset()
getSessionMock.mockReset()
getGroupChatServerMock.mockReset()
getGroupChatServerMock.mockReturnValue(null)
getLocalUsageStatsMock.mockReset()
getActiveProfileNameMock.mockReset()
getActiveProfileNameMock.mockReturnValue('default')
loggerWarnMock.mockReset()
})
@@ -121,4 +134,61 @@ describe('session conversations controller', () => {
expect(getConversationDetailMock).toHaveBeenCalledWith('root', { source: undefined, humanOnly: false })
expect(ctx.body).toEqual({ session_id: 'root', messages: [{ id: 1 }], visible_count: 1, thread_session_count: 1 })
})
it('merges native state.db usage analytics with local Web UI usage for the requested period', async () => {
const today = new Date().toISOString().slice(0, 10)
getLocalUsageStatsMock.mockReturnValue({
input_tokens: 10,
output_tokens: 5,
cache_read_tokens: 2,
cache_write_tokens: 1,
reasoning_tokens: 3,
sessions: 1,
by_model: [
{ model: 'local-model', input_tokens: 10, output_tokens: 5, cache_read_tokens: 2, cache_write_tokens: 1, reasoning_tokens: 3, sessions: 1 },
],
by_day: [
{ date: today, tokens: 15, cache: 2, sessions: 1, cost: 0 },
],
})
getUsageStatsFromDbMock.mockResolvedValue({
input_tokens: 20,
output_tokens: 10,
cache_read_tokens: 4,
cache_write_tokens: 2,
reasoning_tokens: 6,
sessions: 2,
cost: 0.02,
total_api_calls: 7,
by_model: [
{ model: 'hermes-model', input_tokens: 20, output_tokens: 10, cache_read_tokens: 4, cache_write_tokens: 2, reasoning_tokens: 6, sessions: 2 },
],
by_day: [
{ date: today, tokens: 30, cache: 4, sessions: 2, cost: 0.02 },
],
})
const mod = await import('../../packages/server/src/controllers/hermes/sessions')
const ctx: any = { query: { days: '2' }, body: null }
await mod.usageStats(ctx)
expect(getLocalUsageStatsMock).toHaveBeenCalledWith('default', 2)
expect(getUsageStatsFromDbMock).toHaveBeenCalledWith(2)
expect(ctx.body).toMatchObject({
total_input_tokens: 30,
total_output_tokens: 15,
total_cache_read_tokens: 6,
total_cache_write_tokens: 3,
total_reasoning_tokens: 9,
total_sessions: 3,
total_cost: 0.02,
total_api_calls: 7,
period_days: 2,
})
expect(ctx.body.model_usage).toEqual([
{ model: 'hermes-model', input_tokens: 20, output_tokens: 10, cache_read_tokens: 4, cache_write_tokens: 2, reasoning_tokens: 6, sessions: 2 },
{ model: 'local-model', input_tokens: 10, output_tokens: 5, cache_read_tokens: 2, cache_write_tokens: 1, reasoning_tokens: 3, sessions: 1 },
])
expect(ctx.body.daily_usage.find((row: any) => row.date === today)).toMatchObject({ tokens: 45, cache: 6, sessions: 3, cost: 0.02 })
})
})
tests/server/sessions-routes.test.ts
+5
View File
@@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
const listConversationsMock = vi.fn(async (ctx: any) => { ctx.body = { sessions: [{ id: 'conversation-1' }] } })
const getConversationMessagesMock = vi.fn(async (ctx: any) => { ctx.body = { session_id: ctx.params.id, messages: [] } })
const getConversationMessagesPaginatedMock = vi.fn(async (ctx: any) => { ctx.body = { session_id: ctx.params.id, messages: [], pagination: {} } })
const listMock = vi.fn(async (ctx: any) => { ctx.body = { sessions: [{ id: 's1' }] } })
const searchMock = vi.fn(async (ctx: any) => { ctx.body = { results: [{ id: 'search-1' }] } })
const getMock = vi.fn(async (ctx: any) => { ctx.body = { session: { id: ctx.params.id } } })
@@ -15,6 +16,7 @@ const contextLengthMock = vi.fn(async (ctx: any) => { ctx.body = { context_lengt
vi.mock('../../packages/server/src/controllers/hermes/sessions', () => ({
listConversations: listConversationsMock,
getConversationMessages: getConversationMessagesMock,
getConversationMessagesPaginated: getConversationMessagesPaginatedMock,
list: listMock,
search: searchMock,
get: getMock,
@@ -31,6 +33,7 @@ describe('session routes', () => {
vi.resetModules()
listConversationsMock.mockClear()
getConversationMessagesMock.mockClear()
getConversationMessagesPaginatedMock.mockClear()
listMock.mockClear()
searchMock.mockClear()
getMock.mockClear()
@@ -45,10 +48,12 @@ describe('session routes', () => {
expect(paths).toEqual(expect.arrayContaining([
'/api/hermes/sessions/conversations',
'/api/hermes/sessions/conversations/:id/messages',
'/api/hermes/sessions/conversations/:id/messages/paginated',
'/api/hermes/sessions',
'/api/hermes/search/sessions',
'/api/hermes/sessions/search',
'/api/hermes/sessions/usage',
'/api/hermes/usage/stats',
'/api/hermes/sessions/context-length',
'/api/hermes/sessions/:id',
'/api/hermes/sessions/:id/usage',
tests/server/usage-analytics-db.test.ts
+232
View File
@@ -0,0 +1,232 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { DatabaseSync } from 'node:sqlite'
const profileMock = vi.hoisted(() => ({
getActiveProfileDir: vi.fn(),
}))
vi.mock('../../packages/server/src/services/hermes/hermes-profile', () => ({
getActiveProfileDir: profileMock.getActiveProfileDir,
getProfileDir: vi.fn(),
}))
function createStateDb(withApiCallCount = true): string {
const dir = mkdtempSync(join(tmpdir(), 'hermes-usage-'))
const db = new DatabaseSync(join(dir, 'state.db'))
db.exec(`
CREATE TABLE sessions (
id TEXT PRIMARY KEY,
source TEXT,
model TEXT,
started_at INTEGER,
input_tokens INTEGER DEFAULT 0,
output_tokens INTEGER DEFAULT 0,
cache_read_tokens INTEGER DEFAULT 0,
cache_write_tokens INTEGER DEFAULT 0,
reasoning_tokens INTEGER DEFAULT 0,
estimated_cost_usd REAL DEFAULT 0,
actual_cost_usd REAL${withApiCallCount ? ', api_call_count INTEGER DEFAULT 0' : ''}
)
`)
db.close()
return dir
}
function insertSession(
dir: string,
row: {
id: string
source?: string
model?: string | null
started_at: number
input_tokens?: number
output_tokens?: number
cache_read_tokens?: number
cache_write_tokens?: number
reasoning_tokens?: number
estimated_cost_usd?: number
actual_cost_usd?: number | null
api_call_count?: number
},
withApiCallCount = true,
) {
const db = new DatabaseSync(join(dir, 'state.db'))
const baseParams = {
id: row.id,
source: row.source ?? 'cli',
model: row.model ?? null,
started_at: row.started_at,
input_tokens: row.input_tokens ?? 0,
output_tokens: row.output_tokens ?? 0,
cache_read_tokens: row.cache_read_tokens ?? 0,
cache_write_tokens: row.cache_write_tokens ?? 0,
reasoning_tokens: row.reasoning_tokens ?? 0,
estimated_cost_usd: row.estimated_cost_usd ?? 0,
actual_cost_usd: row.actual_cost_usd ?? null,
}
if (withApiCallCount) {
db.prepare(`
INSERT INTO sessions (
id, source, model, started_at, input_tokens, output_tokens,
cache_read_tokens, cache_write_tokens, reasoning_tokens,
estimated_cost_usd, actual_cost_usd, api_call_count
) VALUES (
$id, $source, $model, $started_at, $input_tokens, $output_tokens,
$cache_read_tokens, $cache_write_tokens, $reasoning_tokens,
$estimated_cost_usd, $actual_cost_usd, $api_call_count
)
`).run({ ...baseParams, api_call_count: row.api_call_count ?? 0 })
} else {
db.prepare(`
INSERT INTO sessions (
id, source, model, started_at, input_tokens, output_tokens,
cache_read_tokens, cache_write_tokens, reasoning_tokens,
estimated_cost_usd, actual_cost_usd
) VALUES (
$id, $source, $model, $started_at, $input_tokens, $output_tokens,
$cache_read_tokens, $cache_write_tokens, $reasoning_tokens,
$estimated_cost_usd, $actual_cost_usd
)
`).run(baseParams)
}
db.close()
}
function day(seconds: number): string {
return new Date(seconds * 1000).toISOString().slice(0, 10)
}
describe('native-style Hermes usage analytics DB aggregation', () => {
let profileDir: string | null = null
beforeEach(() => {
vi.resetModules()
profileMock.getActiveProfileDir.mockReset()
})
afterEach(() => {
if (profileDir) rmSync(profileDir, { recursive: true, force: true })
profileDir = null
})
it('sums direct state.db rows in the period while excluding local api_server copies', async () => {
const now = 1_700_000_000
profileDir = createStateDb(true)
profileMock.getActiveProfileDir.mockReturnValue(profileDir)
insertSession(profileDir, {
id: 'root',
source: 'cli',
model: 'gpt-5',
started_at: now - 60,
input_tokens: 100,
output_tokens: 50,
cache_read_tokens: 10,
cache_write_tokens: 2,
reasoning_tokens: 5,
estimated_cost_usd: 0.02,
actual_cost_usd: null,
api_call_count: 1,
})
insertSession(profileDir, {
id: 'tool-child',
source: 'tool',
model: 'tool-model',
started_at: now - 90,
input_tokens: 30,
output_tokens: 20,
cache_read_tokens: 5,
cache_write_tokens: 1,
reasoning_tokens: 2,
estimated_cost_usd: 0.01,
actual_cost_usd: 0.015,
api_call_count: 2,
})
insertSession(profileDir, {
id: 'compress_1',
source: 'cli',
model: 'gpt-5',
started_at: now - 86400,
input_tokens: 7,
output_tokens: 3,
cache_read_tokens: 1,
estimated_cost_usd: 0.005,
})
insertSession(profileDir, {
id: 'null-model',
source: 'cli',
model: null,
started_at: now - 120,
input_tokens: 1,
output_tokens: 2,
estimated_cost_usd: 0.003,
})
insertSession(profileDir, {
id: 'web-local-copy',
source: 'api_server',
model: 'gpt-5',
started_at: now - 30,
input_tokens: 500,
output_tokens: 500,
estimated_cost_usd: 5,
api_call_count: 5,
})
insertSession(profileDir, {
id: 'old',
source: 'cli',
model: 'old-model',
started_at: now - 31 * 86400,
input_tokens: 999,
output_tokens: 999,
estimated_cost_usd: 9,
api_call_count: 9,
})
const mod = await import('../../packages/server/src/db/hermes/sessions-db')
const result = await mod.getUsageStatsFromDb(30, now)
expect(result).toMatchObject({
input_tokens: 138,
output_tokens: 75,
cache_read_tokens: 16,
cache_write_tokens: 3,
reasoning_tokens: 7,
sessions: 4,
total_api_calls: 3,
})
expect(result.cost).toBeCloseTo(0.043)
expect(result.by_model).toEqual([
{ model: 'gpt-5', input_tokens: 107, output_tokens: 53, cache_read_tokens: 11, cache_write_tokens: 2, reasoning_tokens: 5, sessions: 2 },
{ model: 'tool-model', input_tokens: 30, output_tokens: 20, cache_read_tokens: 5, cache_write_tokens: 1, reasoning_tokens: 2, sessions: 1 },
])
expect(result.by_day).toHaveLength(2)
expect(result.by_day[0]).toEqual({ date: day(now - 86400), tokens: 10, cache: 1, sessions: 1, cost: 0.005 })
expect(result.by_day[1]).toMatchObject({ date: day(now), tokens: 203, cache: 15, sessions: 3 })
expect(result.by_day[1].cost).toBeCloseTo(0.038)
})
it('keeps analytics working against older state.db schemas without api_call_count', async () => {
const now = 1_700_000_000
profileDir = createStateDb(false)
profileMock.getActiveProfileDir.mockReturnValue(profileDir)
insertSession(profileDir, {
id: 'legacy',
model: 'legacy-model',
started_at: now - 60,
input_tokens: 4,
output_tokens: 6,
estimated_cost_usd: 0.001,
}, false)
const mod = await import('../../packages/server/src/db/hermes/sessions-db')
const result = await mod.getUsageStatsFromDb(30, now)
expect(result.input_tokens).toBe(4)
expect(result.output_tokens).toBe(6)
expect(result.total_api_calls).toBe(0)
})
})