187 lines
9.9 KiB
Python
187 lines
9.9 KiB
Python
|
"""Advisory timing must remain cheap, scoped and grounded in completed work."""
|
|||
|
|
import copy
|
|||
|
|
import time
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
from fastapi import HTTPException
|
|||
|
|
from sqlmodel import Session
|
|||
|
|
|
|||
|
|
from app import training_estimate as timing, training_map as tm
|
|||
|
|
from app.models import KnowledgeBase, KnowledgeBaseGrant, LLMCall
|
|||
|
|
from tests.test_training_map_background import assessment_env, neutral_plan, request_for
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_estimate_uses_counters_without_reading_vectors_or_calling_ai(assessment_env, monkeypatch):
|
|||
|
|
def forbidden(*args, **kwargs):
|
|||
|
|
raise AssertionError('selection must not read corpus or call AI')
|
|||
|
|
monkeypatch.setattr(tm, 'search_kbs_remote', forbidden)
|
|||
|
|
monkeypatch.setattr(tm, 'model_json', forbidden)
|
|||
|
|
from app import training_corpus
|
|||
|
|
monkeypatch.setattr(training_corpus, 'iter_chunks', forbidden)
|
|||
|
|
with Session(assessment_env.engine) as session:
|
|||
|
|
kb = session.get(KnowledgeBase, assessment_env.kb_id)
|
|||
|
|
kb.chunk_count, kb.chunk_target, kb.file_count = 62000, 2200, 800
|
|||
|
|
session.add(kb); session.commit()
|
|||
|
|
body = tm.EstimateInput(kb_ids=[kb.id, kb.id], mode='full', requested_days=30)
|
|||
|
|
estimate = tm.estimate_assessment(body, request_for(session, assessment_env), session)
|
|||
|
|
quick = tm.estimate_assessment(tm.EstimateInput(kb_ids=[kb.id], mode='quick'), request_for(session, assessment_env), session)
|
|||
|
|
assert estimate['total_chunks'] == 62000 and estimate['total_files'] == 800
|
|||
|
|
assert estimate['estimated_batches'] > 10000
|
|||
|
|
assert estimate['seconds_low'] > 86400 # Do not clamp enormous jobs to five minutes.
|
|||
|
|
assert quick['estimated_batches'] == 0 and quick['seconds_high'] < 300
|
|||
|
|
assert '参考范围' in estimate['basis'] and estimate['history_samples'] == 0
|
|||
|
|
assert estimate['seconds_low'] == sum(stage['seconds_low'] for stage in estimate['stages'])
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_estimate_requires_use_access_for_every_selected_library(assessment_env):
|
|||
|
|
env = assessment_env
|
|||
|
|
with Session(env.engine) as session:
|
|||
|
|
grant = session.get(KnowledgeBaseGrant, env.grant_id)
|
|||
|
|
grant.enabled = False
|
|||
|
|
session.add(grant); session.commit()
|
|||
|
|
with pytest.raises(HTTPException) as denied:
|
|||
|
|
tm.estimate_assessment(tm.EstimateInput(kb_ids=[env.kb_id]), request_for(session, env), session)
|
|||
|
|
assert denied.value.status_code == 404
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_history_only_uses_current_org_provider_model_and_successes(assessment_env):
|
|||
|
|
with Session(assessment_env.engine) as session:
|
|||
|
|
for index in range(4):
|
|||
|
|
session.add(LLMCall(organization_id=1, provider='test-model', model='neutral',
|
|||
|
|
agent_name='训练营导图', status='ok', duration_ms=10000))
|
|||
|
|
for excluded in ({'organization_id': 2}, {'provider': 'other'}, {'model': 'other'},
|
|||
|
|
{'agent_name': 'content'}, {'status': 'timeout'}, {'created_at': '2020-01-01 00:00:00'}):
|
|||
|
|
session.add(LLMCall(**{'organization_id': 1, 'provider': 'test-model', 'model': 'neutral',
|
|||
|
|
'agent_name': '训练营导图', 'status': 'ok', 'duration_ms': 999999, **excluded}))
|
|||
|
|
session.commit()
|
|||
|
|
result = tm.estimate_assessment(tm.EstimateInput(kb_ids=[assessment_env.kb_id]), request_for(session, assessment_env), session)
|
|||
|
|
assert result['history_samples'] == 4
|
|||
|
|
assert result['per_day_seconds_high'] == 60
|
|||
|
|
assert result['estimated_batches'] == 0
|
|||
|
|
assert '最近 4 次' in result['basis']
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_empty_index_has_no_actionable_timing(assessment_env):
|
|||
|
|
with Session(assessment_env.engine) as session:
|
|||
|
|
kb = session.get(KnowledgeBase, assessment_env.kb_id)
|
|||
|
|
kb.chunk_count = 0
|
|||
|
|
session.add(kb); session.commit()
|
|||
|
|
result = tm.estimate_assessment(tm.EstimateInput(kb_ids=[kb.id]), request_for(session, assessment_env), session)
|
|||
|
|
assert result['seconds_low'] == result['seconds_high'] == 0
|
|||
|
|
assert '先完成资料索引' in result['basis']
|
|||
|
|
|
|||
|
|
|
|||
|
|
def coverage(mode='full'):
|
|||
|
|
return {'mode': mode, 'stage': 'reading', 'started_epoch': 1000, 'stage_started_epoch': 1005,
|
|||
|
|
'scanned_chunks': 1000, 'read_chunks': 100, 'processed_batches': 2,
|
|||
|
|
'estimate': {'total_chunks': 1000, 'estimated_batches': 20, 'estimated_reducing_batches': 1,
|
|||
|
|
'stages': [{'id': 'scanning', 'seconds_low': 2, 'seconds_high': 15},
|
|||
|
|
{'id': 'planning', 'seconds_low': 25, 'seconds_high': 120}]}}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_actual_work_progress_does_not_move_with_elapsed_time_or_stage_reversal():
|
|||
|
|
base = coverage()
|
|||
|
|
first = timing.update_timing(base, 1020)
|
|||
|
|
later = timing.update_timing(first, 1400)
|
|||
|
|
assert first['progress_percent'] == later['progress_percent'] < 100
|
|||
|
|
reduced = timing.update_timing({**later, 'stage': 'reducing', 'processed_batches': 5}, 1405)
|
|||
|
|
next_kb = timing.update_timing({**reduced, 'stage': 'reading', 'read_chunks': 200, 'processed_batches': 6}, 1410)
|
|||
|
|
assert next_kb['progress_percent'] >= reduced['progress_percent'] >= first['progress_percent']
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_measured_throughput_replaces_initial_call_reference_after_real_samples():
|
|||
|
|
base = coverage()
|
|||
|
|
measured = {**base, 'timing': {'summary_calls_completed': 2, 'summary_model_seconds': 8,
|
|||
|
|
'reading_calls_completed': 2, 'reducing_calls_completed': 0}}
|
|||
|
|
measured_result = timing.update_timing(measured, 1020)
|
|||
|
|
baseline_result = timing.update_timing(base, 1020)
|
|||
|
|
assert measured_result['remaining_seconds_high'] < baseline_result['remaining_seconds_high']
|
|||
|
|
assert measured_result['progress_percent'] == baseline_result['progress_percent']
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize('mode', ['full', 'quick'])
|
|||
|
|
def test_planning_never_finishes_from_elapsed_and_reports_overdue(mode):
|
|||
|
|
base = {**coverage(mode), 'stage': 'planning', 'stage_started_epoch': 1020}
|
|||
|
|
running = timing.update_timing(base, 2000)
|
|||
|
|
assert running['overdue'] and running['progress_percent'] < 100
|
|||
|
|
assert running['remaining_seconds_high'] > 0
|
|||
|
|
done = timing.update_timing({**running, 'stage': 'complete'}, 2020)
|
|||
|
|
assert done['progress_percent'] == 100
|
|||
|
|
assert done['remaining_seconds_low'] == done['remaining_seconds_high'] == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_failed_and_legacy_progress_do_not_fabricate_a_countdown():
|
|||
|
|
base = timing.update_timing(coverage(), 1020)
|
|||
|
|
stopped = timing.update_timing({**base, 'stage': 'cancelled'}, 1025)
|
|||
|
|
later = timing.update_timing(stopped, 2000)
|
|||
|
|
assert later['elapsed_seconds'] == stopped['elapsed_seconds']
|
|||
|
|
assert later['progress_percent'] == stopped['progress_percent']
|
|||
|
|
assert 'remaining_seconds_low' not in stopped
|
|||
|
|
old = {'stage': 'planning', 'started_epoch': 1000}
|
|||
|
|
assert timing.update_timing(old, 2000) == old
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize('mode', ['full', 'quick'])
|
|||
|
|
def test_assessment_persists_estimate_and_only_completes_progress_after_plan(assessment_env, monkeypatch, mode):
|
|||
|
|
observations = []
|
|||
|
|
def fake_ai(session, row, schema, instructions, evidence):
|
|||
|
|
session.refresh(row)
|
|||
|
|
observations.append(copy.deepcopy(row.coverage))
|
|||
|
|
assert row.coverage['stage'] == 'planning'
|
|||
|
|
assert row.coverage['progress_percent'] < 100
|
|||
|
|
assert row.coverage['estimate']['mode'] == mode
|
|||
|
|
return neutral_plan(evidence)
|
|||
|
|
monkeypatch.setattr(tm, 'model_json', fake_ai)
|
|||
|
|
with Session(assessment_env.engine) as session:
|
|||
|
|
row = tm.assess(assessment_env.draft_id, tm.AssessmentInput(version=1, mode=mode),
|
|||
|
|
request_for(session, assessment_env), session)
|
|||
|
|
assert row.coverage['progress_percent'] == 100
|
|||
|
|
assert row.coverage['remaining_seconds_high'] == 0
|
|||
|
|
assert len(observations) == 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_first_full_batch_marks_reading_before_ai_and_records_measured_calls(assessment_env, monkeypatch):
|
|||
|
|
from app.rag import store
|
|||
|
|
env = assessment_env
|
|||
|
|
store._write_fallback(env.kb_id, [
|
|||
|
|
{'id': f'long-{index}', 'text': f'时间记录{index}：' + (f'按第{index}类任务记录耗时再复盘分配。' * 700),
|
|||
|
|
'meta': {'file': 'lesson.txt'}, 'embedding': [1.0, 0.0]} for index in range(1, 4)])
|
|||
|
|
seen = []
|
|||
|
|
def fake_ai(session, row, schema, instructions, evidence):
|
|||
|
|
seen.append(copy.deepcopy(row.coverage))
|
|||
|
|
if schema is tm.Plan:
|
|||
|
|
return neutral_plan(evidence)
|
|||
|
|
assert row.coverage['stage'] == 'reading'
|
|||
|
|
assert row.coverage['timing']['current_model_started_epoch'] > 0
|
|||
|
|
return {'text': '记录不同任务的耗时和打断来源，按任务分类复盘，并选择具体改进动作。'}
|
|||
|
|
monkeypatch.setattr(tm, 'model_json', fake_ai)
|
|||
|
|
with Session(env.engine) as session:
|
|||
|
|
row = tm.assess(env.draft_id, tm.AssessmentInput(version=1), request_for(session, env), session)
|
|||
|
|
assert row.coverage['timing']['summary_calls_completed'] == len(seen) - 1
|
|||
|
|
assert row.coverage['timing']['summary_model_seconds'] > 0
|
|||
|
|
assert row.coverage['processed_batches'] == len(seen) - 1
|
|||
|
|
assert 'current_model_started_epoch' not in row.coverage['timing']
|
|||
|
|
assert row.coverage['progress_percent'] == 100
|
|||
|
|
assert len(seen) > 2
|
|||
|
|
assert all(item['progress_percent'] < 100 for item in seen)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_progress_get_does_not_autoflush_over_new_worker_progress(assessment_env):
|
|||
|
|
env = assessment_env
|
|||
|
|
with Session(env.engine) as session:
|
|||
|
|
row = session.get(tm.TrainingMap, env.draft_id)
|
|||
|
|
row.status = 'assessing'
|
|||
|
|
row.coverage = {**coverage(), 'started_epoch': time.time() - 2,
|
|||
|
|
'stage_started_epoch': time.time() - 1}
|
|||
|
|
session.add(row); session.commit()
|
|||
|
|
result = tm.get_draft(row.id, request_for(session, env), session)
|
|||
|
|
assert result.coverage['elapsed_seconds'] >= 2
|
|||
|
|
assert not session.is_modified(result)
|
|||
|
|
with Session(env.engine) as worker:
|
|||
|
|
current = worker.get(tm.TrainingMap, row.id)
|
|||
|
|
current.coverage = {**current.coverage, 'read_chunks': 777}
|
|||
|
|
worker.add(current); worker.commit()
|
|||
|
|
session.commit()
|
|||
|
|
with Session(env.engine) as verify:
|
|||
|
|
assert verify.get(tm.TrainingMap, env.draft_id).coverage['read_chunks'] == 777
|