#!/usr/bin/env python3 """Verify retrieval in the isolated acceptance API without changing its data. Only login and read-only search POSTs are allowed. Credentials and response prose are never included in the report. No files are written unless --output is supplied, and credential files are never refreshed or overwritten. """ import argparse import datetime import json import math import re import time import urllib.error import urllib.request from pathlib import Path BASE = 'http://127.0.0.1:18080' ACCEPTANCE = Path(__file__).resolve().parents[2] / 'acceptance' UUID = re.compile(r'^[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}$') MODES = { 'keyword-only': (False, True), 'vector-only': (True, False), 'hybrid': (False, False), } QUERIES = { '1': '产品 使用 说明', '2': '练习 方法 步骤', '3': '沟通 情绪 理解', '4': '关系 感受 表达', '5': '技巧 步骤 学习', '6': '需求 价值 选择', } class VerificationError(Exception): """A fixed, credential-free error suitable for the report.""" class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None class Client: def __init__(self, timeout): self.timeout = timeout self.opener = urllib.request.build_opener( urllib.request.ProxyHandler({}), NoRedirect()) def request(self, route, token=None, payload=None): search_route = re.fullmatch( r'/api/v1/knowledge-bases/[0-9a-fA-F-]+/hybrid-search', route) allowed_read = route in ('/health', '/api/v1/auth/me') allowed_login = route == '/api/v1/auth/login' if not ((payload is None and allowed_read) or (payload is not None and (search_route or allowed_login))): raise VerificationError('Request outside the read-only acceptance allowlist') headers = {'Accept': 'application/json'} if token: headers['Authorization'] = 'Bearer ' + token body = None if payload is not None: headers['Content-Type'] = 'application/json' body = json.dumps(payload).encode('utf-8') request = urllib.request.Request(BASE + route, data=body, headers=headers) started = time.monotonic() try: try: response = self.opener.open(request, timeout=self.timeout) except urllib.error.HTTPError as error: response = error with response: status = response.code raw = response.read(8 * 1024 * 1024 + 1) if len(raw) > 8 * 1024 * 1024: raise VerificationError('Response exceeded the verification size limit') # Denied requests need only a status; never retain their error prose. value = json.loads(raw) if status == 200 else None return status, value, round((time.monotonic() - started) * 1000, 2) except VerificationError: raise except (OSError, urllib.error.URLError, ValueError): raise VerificationError('Request failed, timed out, or returned invalid JSON') from None def read_object(file, label): try: value = json.loads(file.read_text(encoding='utf-8')) except (OSError, ValueError): raise VerificationError(label + ' could not be read as JSON') from None if not isinstance(value, dict): raise VerificationError(label + ' must contain a JSON object') return value def search_payload(query, mode, count): disable_keywords, disable_vector = MODES[mode] return { 'query_text': query, 'match_count': count, 'vector_threshold': 0, 'keyword_threshold': 0, 'disable_keywords_match': disable_keywords, 'disable_vector_match': disable_vector, 'skip_context_enrichment': True, } def inspect_results(value, kb_id, count, mode): if not isinstance(value, dict) or value.get('success') is not True: return {'passed': False, 'failure': 'Invalid search response envelope'} rows = value.get('data') or [] if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows): return {'passed': False, 'failure': 'Invalid search results shape'} scores = [row.get('score') for row in rows] finite = all(isinstance(score, (int, float)) and not isinstance(score, bool) and math.isfinite(score) for score in scores) descending = finite and all(left + 1e-9 >= right for left, right in zip(scores, scores[1:])) in_scope = all(row.get('knowledge_base_id') == kb_id for row in rows) ids = [row.get('id') for row in rows] unique = all(isinstance(item, str) and item for item in ids) and len(set(ids)) == len(ids) # Match types 0 and 1 are vector and keyword hits respectively. expected = {'keyword-only': {1}, 'vector-only': {0}, 'hybrid': {0, 1}}[mode] correct_mode = all(type(row.get('match_type')) is int and row['match_type'] in expected for row in rows) return { 'passed': bool(rows) and len(rows) <= count and finite and descending and in_scope and unique and correct_mode, 'count': len(rows), 'nonempty': bool(rows), 'within_requested_count': len(rows) <= count, 'scores': [round(score, 8) for score in scores] if finite else [], 'scores_finite': finite, 'scores_descending': descending, 'kb_scope_valid': in_scope, 'unique_chunk_ids': unique, 'match_types_valid': correct_mode, } def verify(args, report): credentials = read_object(args.credentials, 'Migration credentials') migration = read_object(args.state, 'Migration state') foreign = read_object(args.foreign_auth, 'Foreign tenant authentication') entries = migration.get('knowledge_bases', {}) if not isinstance(entries, dict) or set(entries) != set(QUERIES): raise VerificationError('Migration state must describe exactly the six expected knowledge bases') kb_ids = [] for legacy_id, row in entries.items(): if not isinstance(row, dict) or not UUID.fullmatch(str(row.get('id', ''))): raise VerificationError('Migration state contains an invalid knowledge base ID') kb_ids.append(row['id']) if len(set(kb_ids)) != 6: raise VerificationError('Migration state contains duplicate knowledge base IDs') if not all(isinstance(credentials.get(key), str) and credentials[key] for key in ('email', 'password')): raise VerificationError('Migration login credentials are incomplete') if not isinstance(foreign.get('token'), str) or not foreign['token']: raise VerificationError('Existing foreign tenant token is required') client = Client(args.timeout) status, _, latency = client.request('/health') report['checks'].append({'check': 'acceptance_health', 'status': status, 'latency_ms': latency, 'passed': status == 200}) if status != 200: raise VerificationError('Acceptance API is not healthy') status, login, latency = client.request('/api/v1/auth/login', payload={ 'email': credentials['email'], 'password': credentials['password'], }) if status != 200 or not isinstance(login, dict) or not isinstance(login.get('token'), str): raise VerificationError('Migration login failed') token = login['token'] tenant_id = login.get('active_tenant', {}).get('id') correct_tenant = tenant_id is not None and tenant_id == migration.get('tenant_id') == credentials.get('tenant_id') report['checks'].append({'check': 'migration_login_scope', 'status': status, 'latency_ms': latency, 'passed': correct_tenant}) if not correct_tenant: raise VerificationError('Migration login is outside the expected tenant') status, identity, latency = client.request('/api/v1/auth/me', foreign['token']) details = identity.get('data', {}) if isinstance(identity, dict) else {} foreign_tenant = (details.get('tenant') or {}).get('id') user = details.get('user') or {} memberships = details.get('memberships') or [] isolated = (status == 200 and foreign_tenant is not None and foreign_tenant != tenant_id and not user.get('is_system_admin') and not user.get('can_access_all_tenants') and not any(member.get('tenant_id') == tenant_id for member in memberships if isinstance(member, dict))) report['checks'].append({'check': 'foreign_identity_valid_and_isolated', 'status': status, 'latency_ms': latency, 'passed': isolated}) if not isolated: report['failures'].append('Foreign authentication is expired, invalid, or not isolated; denial tests cannot prove tenant isolation') for legacy_id in sorted(entries): kb_id = entries[legacy_id]['id'] route = '/api/v1/knowledge-bases/' + kb_id + '/hybrid-search' payload = search_payload(QUERIES[legacy_id], 'keyword-only', args.top_k) for check, auth, enabled, statuses in ( ('unauthenticated_denied', None, True, (401, 403)), ('foreign_tenant_denied', foreign['token'], isolated, (403, 404)), ): if not enabled: continue result = {'check': check, 'legacy_kb': legacy_id, 'kb_id': kb_id} try: status, _, latency = client.request(route, auth, payload) result.update(status=status, latency_ms=latency, passed=status in statuses) except VerificationError as error: result.update(passed=False, failure=str(error)) report['checks'].append(result) for mode in MODES: result = {'legacy_kb': legacy_id, 'kb_id': kb_id, 'mode': mode, 'query': QUERIES[legacy_id], 'top_k': args.top_k} try: status, value, latency = client.request(route, token, search_payload(QUERIES[legacy_id], mode, args.top_k)) result.update(status=status, latency_ms=latency) result.update(inspect_results(value, kb_id, args.top_k, mode) if status == 200 else {'passed': False, 'failure': 'Search HTTP status was not 200'}) except VerificationError as error: result.update(passed=False, failure=str(error)) report['retrieval'].append(result) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--credentials', type=Path, default=ACCEPTANCE / 'migration-token.json') parser.add_argument('--state', type=Path, default=ACCEPTANCE / 'legacy-migration-state.json') parser.add_argument('--foreign-auth', type=Path, default=ACCEPTANCE / 'browser-auth.json') parser.add_argument('--output', type=Path, help='Write the same JSON report to this file only') parser.add_argument('--timeout', type=float, default=45, help='Timeout in seconds for every HTTP request (1-180)') parser.add_argument('--top-k', type=int, default=10, help='Requested primary matches (1-50)') args = parser.parse_args() if not 1 <= args.timeout <= 180 or not 1 <= args.top_k <= 50: parser.error('timeout must be 1-180 seconds and top-k must be 1-50') if args.output and args.output.resolve() in {file.resolve() for file in (args.credentials, args.state, args.foreign_auth)}: parser.error('output must not overwrite authentication or migration state') started = time.monotonic() report = { 'acceptance_url': BASE, 'started_at': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'checks': [], 'retrieval': [], 'failures': [], 'limitations': ['API validation does not independently attest the configured backend or collection', 'Nonempty and ordered matches are smoke checks, not a relevance-quality benchmark'], } try: verify(args, report) except VerificationError as error: report['failures'].append(str(error)) except Exception: # Do not expose response contents, credentials, or traceback locals. report['failures'].append('Unexpected verification failure; inspect the verifier and response shapes locally') all_checks = report['checks'] + report['retrieval'] report['passed'] = (not report['failures'] and len(report['retrieval']) == 18 and all(item.get('passed') for item in all_checks)) report['duration_seconds'] = round(time.monotonic() - started, 2) report['summary'] = {'checks': len(all_checks), 'passed': sum(bool(item.get('passed')) for item in all_checks), 'failed': sum(not item.get('passed') for item in all_checks)} encoded = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n' if args.output: try: args.output.write_text(encoded, encoding='utf-8') except OSError: report['passed'] = False report['failures'].append('Report output could not be written') encoded = json.dumps(report, ensure_ascii=False, indent=2, allow_nan=False) + '\n' print(encoded, end='') return 0 if report['passed'] else 1 if __name__ == '__main__': raise SystemExit(main())