{ "created_at": "2026-09-22T04:38:03.789373+00:00", "protocol": "exact cosine (enable_indexscan=off) vs published HNSW ladder", "primary_n": 200, "scales": [ 5000, 50000, 100000 ], "conditions": [ "raw" ], "comparisons": [ { "corpus_scale_size": 5000, "condition": "raw", "n_questions": 200, "exact_hit_at_1": 0.6, "exact_hit_at_10": 0.795, "exact_mrr": 0.6576, "hnsw_hit_at_1": 0.6, "hnsw_hit_at_10": 0.795, "hnsw_mrr": 0.6576, "delta_hit_at_10_exact_minus_hnsw": 0.0, "elapsed_s": 8.1, "hit10_disagreements": 0 }, { "corpus_scale_size": 50000, "condition": "raw", "n_questions": 200, "exact_hit_at_1": 0.37, "exact_hit_at_10": 0.64, "exact_mrr": 0.4506, "hnsw_hit_at_1": 0.325, "hnsw_hit_at_10": 0.555, "hnsw_mrr": 0.3959, "delta_hit_at_10_exact_minus_hnsw": 0.085, "elapsed_s": 24.8, "hit10_disagreements": 17 }, { "corpus_scale_size": 100000, "condition": "raw", "n_questions": 200, "exact_hit_at_1": 0.34, "exact_hit_at_10": 0.585, "exact_mrr": 0.4084, "hnsw_hit_at_1": 0.305, "hnsw_hit_at_10": 0.51, "hnsw_mrr": 0.3647, "delta_hit_at_10_exact_minus_hnsw": 0.075, "elapsed_s": 43.7, "hit10_disagreements": 15 } ], "runs": [ { "experiment_run_id": "295a6d28-d227-4076-8638-52623b743791", "run_name": "exact_control_mvp_N5000_raw", "condition": "raw", "corpus_scale_size": 5000, "vector_index_type": "pgvector_exact", "force_exact": true, "questions_evaluated": 200, "questions_skipped": 0, "document_recall": 0.7647, "hit_at_1": 0.6, "hit_at_5": 0.74, "hit_at_10": 0.795, "mrr": 0.6576, "per_question": [ { "question_id": "qst_0102::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0002::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0252::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0020::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0113::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0204::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0089::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0006::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0106::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0269::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0025::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0054::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0048::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0181::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0044::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0275::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0317::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0017::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0048::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0060::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0209::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0310::intra_document_reasoning", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0067::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0002::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0350::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0388::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0004::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0344::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0170::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0213::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0368::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0056::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0071::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0116::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1, "rank": 10 }, { "question_id": "qst_0227::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0019::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0047::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0101::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0079::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0426::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0327::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0461::miscellaneous", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0022::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0103::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0234::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0161::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0069::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0145::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0439::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0027::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0418::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0024::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0339::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0285::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0091::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0075::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0470::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0084::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0043::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0422::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0094::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0081::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0068::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0107::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0079::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0076::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0413::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0363::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0149::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0132::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0396::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0435::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0082::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0083::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0037::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0241::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0055::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0282::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0065::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0036::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0297::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0248::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0156::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0096::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0224::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0066::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0042::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0290::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0042::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0458::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0242::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0347::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0200::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0083::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0141::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0445::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0371::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0277::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0322::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0109::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0294::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0284::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0051::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0016::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0259::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0203::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0408::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0126::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0271::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0135::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0065::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0050::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0157::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0240::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0094::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0080::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0404::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0164::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0312::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0215::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0265::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0139::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0369::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0403::constrained", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0194::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0214::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0018::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0175::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0160::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0063::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0062::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0096::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0082::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0010::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0391::constrained", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0288::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0138::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0420::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0462::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0236::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0464::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0373::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0382::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0040::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0323::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0003::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0150::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0134::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0453::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0423::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0038::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0392::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0091::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0389::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0045::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0415::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0045::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0208::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0278::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0189::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0078::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0318::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0003::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0197::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0041::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0166::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0251::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0085::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0448::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0029::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0349::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0324::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0270::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0266::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0247::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0239::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0381::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0393::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0023::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0380::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0124::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0144::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0444::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0359::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0131::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0449::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0162::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0034::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0238::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0057::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0268::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0261::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0021::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0298::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0313::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0244::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0077::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0424::conflicting_info", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0401::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0053::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 } ] }, { "experiment_run_id": "43fd274c-465e-44bf-9a7b-a0824d4bfed8", "run_name": "exact_control_mvp_N50000_raw", "condition": "raw", "corpus_scale_size": 50000, "vector_index_type": "pgvector_exact", "force_exact": true, "questions_evaluated": 200, "questions_skipped": 0, "document_recall": 0.5973, "hit_at_1": 0.37, "hit_at_5": 0.57, "hit_at_10": 0.64, "mrr": 0.4506, "per_question": [ { "question_id": "qst_0102::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0002::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1, "rank": 10 }, { "question_id": "qst_0252::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0020::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0113::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0204::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0089::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0006::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0106::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0269::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0025::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0054::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0048::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0181::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0044::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0275::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0317::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0017::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0048::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0060::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0209::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0310::intra_document_reasoning", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0067::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0002::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0350::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0388::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0004::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0344::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0170::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0213::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0368::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0056::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0071::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0116::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0227::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0019::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0047::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0101::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0079::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0426::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0327::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0461::miscellaneous", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0022::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0103::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0234::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0161::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0069::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0145::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0439::completeness", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0027::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0418::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0024::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0339::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0285::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0091::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0075::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0470::miscellaneous", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0084::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0043::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0422::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0094::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0081::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0068::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0107::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0079::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0076::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0413::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0363::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0149::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0132::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0396::constrained", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0435::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0082::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0083::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0037::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0241::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0055::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0282::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0065::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0036::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0297::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0248::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0156::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0096::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0224::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0066::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0042::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0290::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0042::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0458::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0242::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0347::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0200::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0083::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0141::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0445::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0371::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0277::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0322::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0109::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0294::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0284::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0051::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0016::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0259::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0203::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0408::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0126::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0271::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0135::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0065::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0050::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0157::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0240::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0094::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0080::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0404::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0164::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0312::intra_document_reasoning", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0215::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0265::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0139::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0369::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0403::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0194::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0214::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0018::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0175::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0160::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0063::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0062::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0096::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0082::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0010::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0391::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0288::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0138::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0420::conflicting_info", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0462::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0236::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0464::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0373::project_related", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0382::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0040::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0323::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0003::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0150::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0134::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0453::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0423::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0038::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0392::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0091::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0389::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0045::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0415::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0045::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0208::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0278::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0189::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0078::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0318::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0003::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0197::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0041::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0166::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0251::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0085::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0448::completeness", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0029::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0349::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0324::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0270::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0266::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0247::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0239::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0381::constrained", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0393::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0023::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0380::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0124::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0144::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0444::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0359::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0131::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0449::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0162::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0034::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0238::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0057::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0268::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0261::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0021::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0298::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0313::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0244::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0077::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0424::conflicting_info", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0401::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0053::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 } ] }, { "experiment_run_id": "109cc8dc-58c5-4b0e-9609-998b0c78b71c", "run_name": "exact_control_mvp_N100000_raw", "condition": "raw", "corpus_scale_size": 100000, "vector_index_type": "pgvector_exact", "force_exact": true, "questions_evaluated": 200, "questions_skipped": 0, "document_recall": 0.5398, "hit_at_1": 0.34, "hit_at_5": 0.505, "hit_at_10": 0.585, "mrr": 0.4084, "per_question": [ { "question_id": "qst_0102::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0002::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0252::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0020::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0113::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0204::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0089::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0006::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0106::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0269::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0025::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0054::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0048::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0181::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0044::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0275::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0317::intra_document_reasoning", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0017::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0048::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1, "rank": 10 }, { "question_id": "qst_0060::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0209::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0310::intra_document_reasoning", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0067::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0002::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0350::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0388::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0004::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0344::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0170::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0213::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0368::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0056::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0071::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0116::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0227::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0019::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0047::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0101::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0079::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0426::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0327::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0461::miscellaneous", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0022::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0103::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0234::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0161::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0069::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0145::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0439::completeness", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0027::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0418::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0024::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0339::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0285::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0091::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0075::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0470::miscellaneous", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1, "rank": 10 }, { "question_id": "qst_0084::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0043::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0422::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0094::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0081::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0068::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0107::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0079::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0076::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0413::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0363::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0149::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0132::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0396::constrained", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0435::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0082::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0083::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0037::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0241::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0055::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0282::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0065::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0036::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0297::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0248::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0156::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0096::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0224::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0066::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0042::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0290::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.125, "rank": 8 }, { "question_id": "qst_0042::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0458::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0242::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0347::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0200::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0083::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0141::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0445::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0371::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0277::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0322::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0109::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0294::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0284::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0051::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0016::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0259::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0203::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0408::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0126::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0271::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0135::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0065::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0050::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0157::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0240::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0094::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0080::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0404::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0164::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0312::intra_document_reasoning", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1111111111111111, "rank": 9 }, { "question_id": "qst_0215::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0265::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0139::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0369::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0403::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0194::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0214::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0018::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0175::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0160::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0063::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0062::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0096::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0082::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0010::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0391::constrained", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0288::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0138::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0420::conflicting_info", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0462::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0236::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0464::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0373::project_related", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0382::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0040::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0323::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0003::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0150::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0134::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0453::miscellaneous", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0423::conflicting_info", "hit_at_10": true, "hit_at_1": false, "mrr": 0.16666666666666666, "rank": 6 }, { "question_id": "qst_0038::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0392::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0091::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0389::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0045::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.25, "rank": 4 }, { "question_id": "qst_0415::conflicting_info", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0045::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 }, { "question_id": "qst_0208::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0278::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0189::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0078::metadata", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0318::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0003::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0197::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0041::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0166::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0251::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0085::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0448::completeness", "hit_at_10": true, "hit_at_1": false, "mrr": 0.14285714285714285, "rank": 7 }, { "question_id": "qst_0029::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0349::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0324::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0270::semantic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0266::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0247::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0239::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0381::constrained", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0393::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0023::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0380::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0124::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0144::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0444::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0359::project_related", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0131::basic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0449::completeness", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0162::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0034::basic", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0238::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0057::metadata", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0268::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0261::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0021::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.2, "rank": 5 }, { "question_id": "qst_0298::semantic", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0313::intra_document_reasoning", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0244::semantic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.3333333333333333, "rank": 3 }, { "question_id": "qst_0077::basic", "hit_at_10": true, "hit_at_1": false, "mrr": 0.1, "rank": 10 }, { "question_id": "qst_0424::conflicting_info", "hit_at_10": false, "hit_at_1": false, "mrr": 0.0, "rank": null }, { "question_id": "qst_0401::constrained", "hit_at_10": true, "hit_at_1": true, "mrr": 1.0, "rank": 1 }, { "question_id": "qst_0053::metadata", "hit_at_10": true, "hit_at_1": false, "mrr": 0.5, "rank": 2 } ] } ], "interpretation_note": "If exact Hit@10 still declines with N similarly to HNSW, drift is not solely an ANN artifact. If exact stays flat while HNSW drops, HNSW parameters dominate." }