{ "created_at": "2026-09-24T05:29:16.234865+00:00", "arm": "titan_v2", "embedding_model": "amazon.titan-embed-text-v2:0", "embedding_dim": 1024, "protocol": "exact cosine (enable_indexscan=off, enable_bitmapscan=off) vs published Titan HNSW ladder", "hnsw_source": "artifacts/published/erb_titan_primary200_to100k.json", "primary_n": 200, "scales": [ 5000, 50000, 100000 ], "conditions": [ "raw" ], "comparisons": [ { "corpus_scale_size": 5000, "condition": "raw", "n_questions": 200, "exact_hit_at_1": 0.62, "exact_hit_at_5": 0.73, "exact_hit_at_10": 0.785, "exact_mrr": 0.6714, "exact_document_recall": 0.7663, "hnsw_hit_at_1": 0.62, "hnsw_hit_at_5": 0.73, "hnsw_hit_at_10": 0.785, "hnsw_mrr": 0.6714, "hnsw_document_recall": 0.7663, "delta_hit_at_10_exact_minus_hnsw": 0.0, "delta_hit_at_1_exact_minus_hnsw": 0.0, "hit10_disagreements": 0, "identical_top10_lists": 200, "elapsed_s": 6.4 }, { "corpus_scale_size": 50000, "condition": "raw", "n_questions": 200, "exact_hit_at_1": 0.425, "exact_hit_at_5": 0.605, "exact_hit_at_10": 0.645, "exact_mrr": 0.504, "exact_document_recall": 0.6148, "hnsw_hit_at_1": 0.425, "hnsw_hit_at_5": 0.605, "hnsw_hit_at_10": 0.645, "hnsw_mrr": 0.504, "hnsw_document_recall": 0.6148, "delta_hit_at_10_exact_minus_hnsw": 0.0, "delta_hit_at_1_exact_minus_hnsw": 0.0, "hit10_disagreements": 0, "identical_top10_lists": 200, "elapsed_s": 20.1 }, { "corpus_scale_size": 100000, "condition": "raw", "n_questions": 200, "exact_hit_at_1": 0.385, "exact_hit_at_5": 0.565, "exact_hit_at_10": 0.61, "exact_mrr": 0.465, "exact_document_recall": 0.5769, "hnsw_hit_at_1": 0.385, "hnsw_hit_at_5": 0.565, "hnsw_hit_at_10": 0.61, "hnsw_mrr": 0.465, "hnsw_document_recall": 0.5769, "delta_hit_at_10_exact_minus_hnsw": 0.0, "delta_hit_at_1_exact_minus_hnsw": 0.0, "hit10_disagreements": 0, "identical_top10_lists": 200, "elapsed_s": 37.5 } ], "runs": [ { "corpus_scale_size": 5000, "condition": "raw", "hit_at_1": 0.62, "hit_at_5": 0.73, "hit_at_10": 0.785, "mrr": 0.6714, "document_recall": 0.7663, "per_question": [ { "question_id": "qst_0102::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e044274a9d184700bb8cdf9ddb6d9d02", "dsid_9384397e3c294b0a80d0e74d12178c1f", "dsid_ca5e4082206c4ef087c7008f42d5836a", "dsid_b35f08478f9a48f3b04f8a36ac3b158d", "dsid_64701ef21edf465081b28ae04db2aedd", "dsid_0fce8ede88904404b50d1be778f6a4e3", "dsid_553e1651575a416dae6b92d019d2f9f4", "dsid_3dd41f939c6d48c7af98b50d6ee2e2d5", "dsid_90ab88344489468887805a89b3438f67", "dsid_a3f94972391244f38ef773c05926af83" ] }, { "question_id": "qst_0002::basic", "question_type": "basic", "rank": 7, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.14285714285714285, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8eaa2785974140b3894297324977a8ff", "dsid_4b2ac6984bdf4737a18520e158b04453", "dsid_857e8bf5ab2b423e9bc8bd5f6ea1583c", "dsid_6849218bed8c4f169cb6dacfc33a7c4f", "dsid_15a9d93cc59c4241a97a7e72f749a216", "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_9550250a59e74f1bbd5612480b2e7100", "dsid_b885123ef2c04d508ee071328a4e5aa7", "dsid_d360ef464ece4cacb0c3a98b55733f02", "dsid_a98ada14af2142a7a46266584d05c846" ] }, { "question_id": "qst_0252::semantic", "question_type": "semantic", "rank": 10, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.1, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7b4d5aae7f064e25aa4e0138a6c9d4d0", "dsid_d8e75effbe37456183c936ee38d3a302", "dsid_e964a683919b4fd9ba304c8a976d8129", "dsid_981f4a0281054f9aaaec7887156eebc4", "dsid_7bed0739a8f241219e4f3ab0304916bb", "dsid_d16a7fb54f43425f9c0fd5038ddf4816", "dsid_99fa70fd8ffc4298bb1dfcba1671c051", "dsid_b360d00be3f84b08b1a879d102835d2f", "dsid_1f1cc45f2efc4169a5c50092e7c0f6c9", "dsid_01eaeaf6045941beaeaf74e6170aceea" ] }, { "question_id": "qst_0020::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1c53affffb4f477ca476bec3138b60eb", "dsid_0e0ac7276c864c63a02586fa07ae85e1", "dsid_2df1f31be99d4ebc87fcb7dc3cbc3d86", "dsid_852961d22e0e48b194d7a3f8ccf17173", "dsid_c74b23d6f05f4376a5e583a83585217c", "dsid_78f566479b3b490eafa3386a0a4c6b1d", "dsid_f3661d10671a4277ab9ee5e053d19e18", "dsid_8779ac72d6504759b701178632772f6c", "dsid_7c4b6ad8c8544611af8dda42b7dfb081", "dsid_49be19b1f6d24b63aa33725775fa04f0" ] }, { "question_id": "qst_0113::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c53d4b486a1e485186a7e4b9d1c3f3ef", "dsid_6e558585f60b4644bf54e2e558f481de", "dsid_69a2bffadbcc44ecb12e6ad8f13807de", "dsid_4255f3f005a44e8e84d7c8e10e6be012", "dsid_6807c651b55546559733c7c6f8dc133d", "dsid_59875becee60430e9b4c9ed7c5be168c", "dsid_2bf2ca0fb29f4da99557b3a8e6e019cc", "dsid_199643d1a8244748972aed03e99257dd", "dsid_2b7848a8491749f8b9b9a8c13f78445f", "dsid_e28ebeaadb9f4a6bbd01fd4d99414a8b" ] }, { "question_id": "qst_0204::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_acd812cda19c40fa8c5a9c044aacd6d6", "dsid_52a32f2d0cda4647b7fa12dda76ed277", "dsid_033acb55079b4d42aac8ecc7d6b3b511", "dsid_799a1bb8e7234cdf89834a0a17e04ba4", "dsid_c8ee7fabdcb946dc842d40c552180329", "dsid_2b567272362e4f06a64ce6c6bd987e40", "dsid_fff0eaf808b149ae93f73d4d56dc2dac", "dsid_25b3d349510d453e90b9d34d541ee27c", "dsid_d2d62518de7849f8b62097aa09be7701", "dsid_6df6cca7e7124f20850ebe9077fd5958" ] }, { "question_id": "qst_0089::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d7953f120c574f71873c6abee98e7ce1", "dsid_426ca69578a84dd4af512e5a2782086e", "dsid_f6ef4260feac48f2bb59f35bcd8f9102", "dsid_c8eb6a1c014841bdaa2e40fefcd4ac08", "dsid_22a7e9af2760474dbdf611c2c21cd231", "dsid_00f81dfe95d641e897af85983a14225d", "dsid_5defc84854074efcbc2ff5c0d2842b76", "dsid_82122f8103524e75972fc64ffde2c6ac", "dsid_f123f590fdb9457693c302a088b050bc", "dsid_5c5fb2911bf347b79e23d5d166e7a5df" ] }, { "question_id": "qst_0006::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5b3278541a824fc7a21d24dc2d7bd5ba", "dsid_184be937d34a412ab5e61366d54d8ed6", "dsid_33a9be3f84b647dd8fc211b7a3ef5b40", "dsid_a98a3028b9f54909b8f4684b2f77dd35", "dsid_c0c93abadf4c438189e23cb45256c6e9", "dsid_865449ae95b044768f33773263bfeaad", "dsid_5f601f0ce17147648370bfde2d9392d1", "dsid_a52a2354f2ef4a26be3e96dc2f9d6a63", "dsid_4c46ea65a2654b94b21d690327c3727a", "dsid_639058015a4e4442927f86b951ba5b44" ] }, { "question_id": "qst_0106::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6f69a170c5ab486588cf0bd92e0d53db", "dsid_717ef1061dd54169810ae439dcffb63f", "dsid_8692d8905d2e4364b437f968c185ebab", "dsid_5aff8d8242e34bf5b2a5009692149e37", "dsid_f3a36f7c3b8646e7b6ac24607c9cd49a", "dsid_d81ca6c0b6104ad78c27a3b2cebecce5", "dsid_7328c766c5994fd2ad4a12e883570860", "dsid_5db9e7b87517429ca2ade855d6d5a976", "dsid_f5030a0cddaf463296c0eb918e7a78e1", "dsid_e42d7794db734f7fb69299207974d453" ] }, { "question_id": "qst_0269::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fad07fd45c964078ba757dfd8d79ee51", "dsid_aa32fc5e084a4a35983bebb576693b7f", "dsid_88b63e0ab08844f68167e2350038f059", "dsid_97e17aff22b744e48ca567f013e5e8e7", "dsid_f17df5d7a13f48028e27ea8c10d1c7a9", "dsid_821022eec9ff4aafbbbb9707ccbfd7c9", "dsid_acc6cfd3ab3549ef955ad8dbd34af834", "dsid_b3bb0d7e74a749988536933d7419a178", "dsid_557c47d8aae543afa40b6109b257f863", "dsid_c83811391709468fa9534e6495c205e3" ] }, { "question_id": "qst_0025::metadata", "question_type": "metadata", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_4700682214424a7aafe594325c48459b", "dsid_4a2ce3e1be4d4f00a089ccc00ac92b8c", "dsid_a98ada14af2142a7a46266584d05c846", "dsid_64212a1689814ac19deff834ce203ebd", "dsid_19b05c958acb478a90cab25257366a29", "dsid_c8a4367928d447d0bc86535046bd2a04", "dsid_269469b1441c415da28b7dacabf9282a", "dsid_e9a84585846c4f3e80c809d21ce9520a", "dsid_d28c5ac4758742c5b293a8869f5d7755" ] }, { "question_id": "qst_0054::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_de78a2c04c4e4ac397f3fdbca2647dbc", "dsid_221c49190a14450595ca5dd9ab1d801b", "dsid_0cfe1a14a8a242fdafefd2c67210aa23", "dsid_0bac9ae36dc44c4d981f8b7c98f30d48", "dsid_1466ca42d6a5487dbbcf613177519e72", "dsid_3025c9421e1c4bff9f8b60262c93f8ea", "dsid_f5030a0cddaf463296c0eb918e7a78e1", "dsid_3c9fe2220c204f13b878bd7d3a168f91", "dsid_d8480d7f1f7344c28b72853b35f9330d", "dsid_1ba2ec9ec75541db829f35747a23dc22" ] }, { "question_id": "qst_0048::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f3af4a0db68c4b15983439460cc73006", "dsid_a3954582c60442ddb1924caa03bed22e", "dsid_8db16dd149b74972ae947d5525043b45", "dsid_8a085e12606b4ea79f43ee176cba2ccb", "dsid_5e1a69f78f44438596a2e44f15f0ce57", "dsid_96912c22b0b04024b573606105bd5ea0", "dsid_25082e145cad467d934af9f62b4093a4", "dsid_1c9140e818384efca371611858fffcdf", "dsid_70f3865fb0b84b7e879497718c0dd027", "dsid_fc356e32e9a64c3199be174cb91432eb" ] }, { "question_id": "qst_0181::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ba16d5ba48764a7cbd00c69b344d27f6", "dsid_21e8152aa32044e8bfc1f3ffbf1c3a4a", "dsid_a3eefbb375474efc91283e84915b132b", "dsid_1a856cfde7344e08a209e61e44b8f884", "dsid_41d08aec62844f0683786ff59933b1f4", "dsid_e4b87295de784676aece27d99728260b", "dsid_e522d035f5ab4e4aa1812653a0b654aa", "dsid_f562ffd4243045689cac6aa9c92e465c", "dsid_972b703ee7264725af87afcab8d65c52", "dsid_a4417244212a459fab8ead17d8a58005" ] }, { "question_id": "qst_0044::metadata", "question_type": "metadata", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_033acb55079b4d42aac8ecc7d6b3b511", "dsid_52a32f2d0cda4647b7fa12dda76ed277", "dsid_6ee6ef4ce1c84921aec1a07b506541fb", "dsid_acd812cda19c40fa8c5a9c044aacd6d6", "dsid_de9c1d389e964346a5d863550ba817a2", "dsid_676625b7b7064b30a00da4f53fd24f88", "dsid_c0d9c3e9a3be4330bba855d26387e0ec", "dsid_4a8247d18bb5495e8b5c3476ffa5d4fa", "dsid_c887f6ded8174824a9b88620e603924d", "dsid_d2d62518de7849f8b62097aa09be7701" ] }, { "question_id": "qst_0275::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a55032c4c2064671a4b0abf975438e1a", "dsid_48fa74f556aa4f0c9996b53170693e8f", "dsid_fcffa55108304ac59a84209b17a160c3", "dsid_1dc182f68d6642cabbf055248ed2f1a4", "dsid_1732e93e187f4b5eb6352e1ae9eff5fe", "dsid_c0dd7c8e00a04b0dbc7961839f54790e", "dsid_58692c6c53584440b9efb9160d644838", "dsid_16af07ca08904644989072866aab7b6d", "dsid_01a0761749164c4590a0da99c43192f5", "dsid_46a4cb87db414e769f2df86f01626948" ] }, { "question_id": "qst_0317::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_36c9d47041b44075810c5582f1a6b3fa", "dsid_8fe92e57ba9640bf8d2044a78e8f78d7", "dsid_0f34f8c7a11548c39e32c51e2c2f8118", "dsid_a52a2354f2ef4a26be3e96dc2f9d6a63", "dsid_d32eb5051b554e038222c3bf399acef8", "dsid_47a31b8f53cb4405b09e3550b853de64", "dsid_75bc2477d775437fac0690975036c78e", "dsid_3242990ca4ca4b139b628fb65887ddfc", "dsid_e0cb5cf4cc20458d957c10a5f1d29a31", "dsid_6e4c61bb6a5e4746a5b7b79557f3d7e4" ] }, { "question_id": "qst_0017::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c83811391709468fa9534e6495c205e3", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_d9d0ca701cc54d58adae135f69cc83d7", "dsid_1a3a408d31b649cf8597fd1fefbc24fc", "dsid_b837e1140d1a4c3d8efd2f42d9b6a799", "dsid_ef048583ee2a41ceaf97142a4ad3550c", "dsid_981f4a0281054f9aaaec7887156eebc4", "dsid_85ade33e53334639be778b9c7aabe637", "dsid_5555c6b607ef4785bf8df9d0970f743a", "dsid_0d308fa6f2b54bfbb13a298c9fc8ced0" ] }, { "question_id": "qst_0048::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8412db85d95a45f7bbfb14dacedb4d8b", "dsid_64701ef21edf465081b28ae04db2aedd", "dsid_3b39f67ce23a4de59fc861fdab86d9b9", "dsid_a3f94972391244f38ef773c05926af83", "dsid_7012ff9881064ce09dfe4034055cc662", "dsid_5139b75526bc4cb0a55b63eaecebe105", "dsid_a05d44339649492fa7c0ec056dffbdf8", "dsid_90ab88344489468887805a89b3438f67", "dsid_be7e5cffb2704eb0b90de833ef6be6c1", "dsid_e81ac10a5db6438996d2ddc78e9a9d6d" ] }, { "question_id": "qst_0060::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_e5fe3fd0cf444668a64970ec00818f46", "dsid_8b56f59b3a7744e3bb4b429912ad5aa2", "dsid_7bed0739a8f241219e4f3ab0304916bb", "dsid_f5a0ffce1a574f28ae8656fd6d04ddb9", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_78001368bf0e456ea8a4d0417b2c5b01", "dsid_85ade33e53334639be778b9c7aabe637", "dsid_f9fb55aaf7014deba1ba303a63bf5e7d", "dsid_5555c6b607ef4785bf8df9d0970f743a", "dsid_1878e91704a24813a12f69b40444f168" ] }, { "question_id": "qst_0209::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_58a34b025e1c4ccf8c97dbc9e042548b", "dsid_715031e81f3f436fa0282fd0f248a716", "dsid_3c69e5d0b7a6462d9c7d54f40dbb914d", "dsid_e6f4d2efb6ab4fa0827a09b203e5e1e5", "dsid_d9a7de9e7cfc4fe0b7b57765b6d25ad9", "dsid_e19b6b7d1ae04f4287fda338801ed824", "dsid_e364e6ecabe04057bdb31600d0343557", "dsid_08079847ab4048bbb73790caff3e0fcb" ] }, { "question_id": "qst_0310::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a5d293df3be843caa748deb7c0946abf", "dsid_9c63777c789c4f7db9e391024dd887ae", "dsid_0950975e9dae4a24963d444abe13dc30", "dsid_ab40d0c5ecac436ea8df60f231a90f37", "dsid_ebd7217aa2174a67a3646fedeb72ebaa", "dsid_b9025f196ac14b2cb7a2905654dd42d7", "dsid_3c643d2e76d94a2594d87433bc7937ac", "dsid_b31de0eacd2a4b1fb8eb710d22396bb8", "dsid_090167b6b6424b3d88e1cf137c14dc33", "dsid_546f40265e5740b583eb9f24ffa53570" ] }, { "question_id": "qst_0067::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_14a447cb5b1140709c7cb3cc9a88c430", "dsid_5204d2a9973a49bbab1ea7a9fb129503", "dsid_aaddf460f19349d788929b9256754451", "dsid_932d57f39e9242d7955f63a2ad49b395", "dsid_5eac201552f8445998948b63bbb2210a", "dsid_d9949dde638b462f8db3ff1e4a1c3a1b", "dsid_c27483a128094c9c8cef4eef00074763", "dsid_fdc30024e874443eb02ce1fe78bde28f", "dsid_1561a2bb12db4f1e926b2f13e84a1a73", "dsid_c9435a0a83724528bfd39f530d0c972a" ] }, { "question_id": "qst_0002::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_dfc72173b67f4ce4878654671940a352", "dsid_62cfacd38cef4cc98121a2581decda5e", "dsid_3581b147f8954835845909742b0d7aa2", "dsid_05faa1ab6427403888a06d7083c7f0a9", "dsid_bd58517ef13a42428e5d14407902df5a", "dsid_d30e78dfb360420e81ccd798d1ad160d", "dsid_9a6ec49dbe324effb0b66981c61eb5ad", "dsid_1dceec2e5ce44dd989b690dc690ef9f2", "dsid_8e215f0af2f640eab127aa0fc0bf6023", "dsid_d295b7207fec4a6fb656f3e2a565d4e6" ] }, { "question_id": "qst_0350::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f3e6edec6f4947a5bd5aa7fd557d7d2b", "dsid_939eea1a52f84393b0f87492b9aba8c2", "dsid_7319adbdfeb94cf9a955a23092938026", "dsid_2f1f0358b85148f19d38f619891bfb15", "dsid_b6993ecb4572479fa0c8f347da254b58", "dsid_38e7aac0d2ed4545b7a92c2c441825a1", "dsid_582cf753660344acb9d061d282499d4b", "dsid_a7baddf163ef4946a97b4765bf1cf08b", "dsid_08a58024d74d4b91b31f1ab5a6efa640", "dsid_95c7aa5238f94570a8e0a8684736b666" ] }, { "question_id": "qst_0388::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8c15ff526e00491e9ccbde5ef4a4458c", "dsid_269469b1441c415da28b7dacabf9282a", "dsid_c8a4367928d447d0bc86535046bd2a04", "dsid_d7b6b915ba6a47d98fab7dddcc43e50c", "dsid_ceff4fc0c1da46d7825bca62bda34df2", "dsid_3ac2c680a56a44e39ddcfd257804de28", "dsid_f226b067355f408ab8c6f59f3bf8a24f", "dsid_3db759c2bbc04576bad02e581e243007", "dsid_92a8459aa8704d299750dd96432c9bca", "dsid_57d9191873854e67a46987df5f503dfb" ] }, { "question_id": "qst_0004::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7d9551aa21124d70b0e247c390d88622", "dsid_6c4c1c875e704f09b4d791d64d7bc7e5", "dsid_e7dd8aa445994a6382c0ebeac6151bef", "dsid_96151fd86fcd4aef83e13a706768dd11", "dsid_4a1ec9242b1745ec96f51861a127e53d", "dsid_537dd80cd7e24250af972f6b3563281c", "dsid_33c31a746b7b459faff2c646ec5380a1", "dsid_503e32dcac7a4a249e1ab0dabe5f9e2a", "dsid_98cbf76fdf33419eae7857380ea28683", "dsid_99d208be71564a75b30e4b045f4712f0" ] }, { "question_id": "qst_0344::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_4b2ac6984bdf4737a18520e158b04453", "dsid_409618a3958944f9b78362ed7e01bb33", "dsid_a5a5235d973e421c870e11ceea4887ed", "dsid_ef048583ee2a41ceaf97142a4ad3550c", "dsid_51895b7e9f2942898291b66b72886b15", "dsid_e33c122264214846a8b7df7701d31482", "dsid_4eac2bf15bc248739000b538834886f2", "dsid_31638228bb754a38bf38d20c25b32bda", "dsid_889a3085065142d5a21a708511ef55f9", "dsid_113f5cc6931640c3b82adcc07142d829" ] }, { "question_id": "qst_0170::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_3c69e5d0b7a6462d9c7d54f40dbb914d", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_77de6e86752245b29ce92f7009728206", "dsid_2a94754480884dd6a23c3080e4a98bef", "dsid_f82615c63ad6463693e54efacca49006", "dsid_715031e81f3f436fa0282fd0f248a716", "dsid_b65f12db881f41dc9239fe044f4754dd", "dsid_e6f4d2efb6ab4fa0827a09b203e5e1e5", "dsid_ad14c3cb06c4494093dfcfff5098d89d" ] }, { "question_id": "qst_0213::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ad14c3cb06c4494093dfcfff5098d89d", "dsid_8d95e02dfe9241b78cf697c0b4921c9b", "dsid_b65f12db881f41dc9239fe044f4754dd", "dsid_7d3e5ffd4c6e493aba47c435443e9a06", "dsid_77de6e86752245b29ce92f7009728206", "dsid_f82615c63ad6463693e54efacca49006", "dsid_c3e4887538a04a36b9581aad7a3e0fac", "dsid_f9f0b66621ec4fa8a8a9688090836c99", "dsid_1bfd5cf3ae504f46b081ff3082c99b6c", "dsid_53ed9962267d4e2989730f0be907b9a5" ] }, { "question_id": "qst_0368::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_abfb68dde21f4fd2aa7d70c472447f89", "dsid_18ac88b8d1ce4470a99abeebfca95cd4", "dsid_2e4183d040b341cf843366e2c15402af", "dsid_0f9c8d82259c4d5a95bc5865fc492919", "dsid_d9984be4f79f4f3587b02b29635fefc9", "dsid_10fc70076336485483c088238c7d144e", "dsid_b875b2c7d7404486832f7cf8b25ed483", "dsid_e03720dca105408889af286a43f61744", "dsid_f908cdbbf2e24c8d8fe90f15ed9236fa", "dsid_8f610770a16d46d4b6aedfb764ebd3a6" ] }, { "question_id": "qst_0056::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_821022eec9ff4aafbbbb9707ccbfd7c9", "dsid_1b6a17dd05f64e97abd48d43d7e49cdc", "dsid_9c63777c789c4f7db9e391024dd887ae", "dsid_80b040ef3b4a47b082062e490dd845f0", "dsid_42ba23bfa9ce4076b3c3c44fb5029fb4", "dsid_6a8960473c834e90937c940c221bf958", "dsid_b3bb79929639401798f43221b1671498", "dsid_33a2b9bc45fb41d09e20cebca46de8a3", "dsid_ed968727168a411ca8628448b671c699", "dsid_a2dc7d066f1a4b0cab825a3bd07d0fec" ] }, { "question_id": "qst_0071::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d11427550fd14a8fb8335ef28fca5176", "dsid_41af3c8f60724a4ba6210606cb4e3617", "dsid_9f5084c3536545bd82d81d880a49cf67", "dsid_236084252e724fbbb3b21533579208fd", "dsid_ee9942076dfa4249a77338a401d25eab", "dsid_57a250dec9414edfaf0460f217f0700a", "dsid_13eefcef2cb84de6b5b25f7efe85664b", "dsid_512c80bd4c654348b294c8a63c2f0c9c", "dsid_a4abf484b679498096d46b21d91b523f", "dsid_28a0ac3c2505410eb3e0e84468a29f3f" ] }, { "question_id": "qst_0116::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_0032ae6f295b4845b21638f56eae59a2", "dsid_69216a68434247baaaadb717e80636d4", "dsid_6933f9241ef140da9d3bf98be52be867", "dsid_89f482fcee1e4a0da0ffb8d8e7760ac9", "dsid_b793b06789ae4147884e94681c6beae1", "dsid_5f0ab47fe8de47a89bb6668f8722becf", "dsid_da0e55a5772945deb0f611f5d1d25c16", "dsid_e91e68b31a6f4935b87d6f74dd07e1f6", "dsid_d30e78dfb360420e81ccd798d1ad160d", "dsid_67a8c3300752435296938175e9a1daf5" ] }, { "question_id": "qst_0227::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_48147d8287d04e928ecaf0bd7b35859a", "dsid_32d162435ad34eeca3861eeca4463879", "dsid_15a9d93cc59c4241a97a7e72f749a216", "dsid_d7b6b915ba6a47d98fab7dddcc43e50c", "dsid_3f068bf8b9844100bfb79e5ec7f58c03", "dsid_92a8459aa8704d299750dd96432c9bca", "dsid_4f0b59f1c83640459247deabe08dbf23", "dsid_4f1b354d79294435954b024bdf0dfd2e", "dsid_4a2ce3e1be4d4f00a089ccc00ac92b8c" ] }, { "question_id": "qst_0019::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e728a420a02a4ddcb125e6adb09d211e", "dsid_fd184cac01d44876afdf57705ff2b3e7", "dsid_5e438f989083490082a377dc566a2a45", "dsid_59cb294da8964683bb165a3bb615df71", "dsid_e20c31eb5466411ab6c7be17d08d02f5", "dsid_f151ee037d5a4f538e410ade668e6f20", "dsid_a277ed2dc74c4bad82bc71b534b6356f", "dsid_b5fd84b12112401a96f7a6333ac51ece", "dsid_95537acb247044fd95aa784cc535878c", "dsid_c0c93abadf4c438189e23cb45256c6e9" ] }, { "question_id": "qst_0047::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7a2cfe4b7e574e6995f33e64bbdbf426", "dsid_7ab12a0c235947b4a69264e27b2dcf07", "dsid_c110d111efc447e78fcc15d6dca92af8", "dsid_e54ec48a2f3f4ff68589a34a7b387f79", "dsid_89c0c31e220c4777b15c3c6ff0eff61d", "dsid_e528bed025b44f71bc37a54f5a9aefcd", "dsid_0a02b625a9dd4681820dc542a99bc43b", "dsid_deb522245fa341638a0125628c263a0d", "dsid_3c58103407de4e6a911e9f8262b9dce9", "dsid_c3e4887538a04a36b9581aad7a3e0fac" ] }, { "question_id": "qst_0101::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fb6eb53e8969406d844868b30ac46b20", "dsid_cd74a1852f714974878e3366af7b1e29", "dsid_565f801f14bd414799804ab3f374cd46", "dsid_d44fd3d3718743b09299126264d88800", "dsid_7e6ff21b5ef74effa08976e7a14cd1c6", "dsid_c3c354c0d29142f4819cf38c14b8b444", "dsid_06f6fbc4c86c485fa1ebba45ce91d92b", "dsid_72acbfa2791742cbaa22e34bacaee53a", "dsid_d323c60879174cc984df28817cde6bb5", "dsid_0517272f49c544fbb3eb01799da8dd82" ] }, { "question_id": "qst_0079::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9776bd75ffbb4ca7b10669be1512daa3", "dsid_4862d9d3c9894edf97f2449bf0540ccf", "dsid_dc8da53aae4c4a81be13ae8e519cd42d", "dsid_2f1f0358b85148f19d38f619891bfb15", "dsid_b72133b826ef4dbf833b558aa9e7b4f6", "dsid_205c62f1ace8402abf9820662d82ccef", "dsid_9f1a05a6016945148a6aa988f0934a19", "dsid_803c51d66ba54d74868dccdd187e933b", "dsid_3c3de6be742b4ce0876b9dc9fdf7a99f", "dsid_41457ac2ecd74f088567aec87a7cd003" ] }, { "question_id": "qst_0426::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c90bd544f3b5437fba711492db31aeea", "dsid_553e1651575a416dae6b92d019d2f9f4", "dsid_83effe9968594d25a5ecb719178034c6", "dsid_92219f7ec56b4e2fa0a83190d8ccec3f", "dsid_96070e9bccf74b43bcde8e0e934d2428", "dsid_c525c744b8fc456a9f9d74c21b84ea32", "dsid_f6fa144362d7412c96d8d4798bda758a", "dsid_efbce837939d45368daa696138b58414", "dsid_98072730599c40f2a0b4bdd6ef5c09c7", "dsid_1608b1297c804cedbcb06c66ca15291e" ] }, { "question_id": "qst_0327::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_69e745b76aa34eab8721ba0c88482ec0", "dsid_c52c609c0cdf4211a7e8c57bb1da72e4", "dsid_37befab212ae4ed9be7ebe612d942658", "dsid_2615e17a95fb4ccbb07d872f13608ce9", "dsid_d7aa5bdaebe44ce2a2401a1568f27a08", "dsid_386066d943354e6592a86fceb2c46a62", "dsid_4c9ec1255c4544c0941d71fc16688b9a", "dsid_f952b60df24d4736b831a643993aee67", "dsid_782d6b244dce4e5c8c9e21822c7bef49", "dsid_63ff9a2868b444c897f317d9849b94d9" ] }, { "question_id": "qst_0461::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_22b0987227cd4fdb94d67e5a62bbde40", "dsid_0fa69b4e576f417ab759af8e33cb049f", "dsid_c5288dd4874345adb80f4a71c9a18773", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_047e2ba9fd4e42d09be10391a6448d90", "dsid_a5d293df3be843caa748deb7c0946abf", "dsid_896272184a6845a19d9bd1a0a4e3e671", "dsid_8d33dfb27da14dc793879351f2eb743a", "dsid_d02dc8d4929d435e9ed5f2ae716f9697", "dsid_fbcdb6891aea404da7f2f5b40163a294" ] }, { "question_id": "qst_0022::metadata", "question_type": "metadata", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b4443ecc53154934902652d496c9d12e", "dsid_f94b93a4bc7948ddab45c371187dc6c6", "dsid_b17470b55308417887920a77bc20a1b5", "dsid_bfda525a37804b11a62e89da623e90ff", "dsid_cc6747e10e8c40699f5adcffaf3a667e", "dsid_a3954582c60442ddb1924caa03bed22e", "dsid_505b74e43e9a4117bcd9559b912cb449", "dsid_0882f65ac06243a5a4186c10755b0ece", "dsid_e8d57d5407b346e3ab2f4e15ead4d12b", "dsid_feb0771faeee41209acbb59e27adfe69" ] }, { "question_id": "qst_0103::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_47fa00c889fb4446b0481efea384a7ce", "dsid_6a5dde2bff494b6a9989dff82d4c4644", "dsid_1a4dc2c6f4484058bebc7c22da836131", "dsid_0831ceb4c5094699ac459d2c13d27d90", "dsid_87fb9790b17347268d4e0672af499c40", "dsid_4c67cda7812440b6abfb79e6455930db", "dsid_008298ea4fcb4b979c4af5db2458e4d5", "dsid_ccae497429c340f3b051a689f945ba7b", "dsid_c1533fbf241240629c9e035c4f59cc7e", "dsid_f952b60df24d4736b831a643993aee67" ] }, { "question_id": "qst_0234::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b16eb801dbc741558b841c7c3047e98d", "dsid_512c80bd4c654348b294c8a63c2f0c9c", "dsid_4ecc265a59c5499da505794b95399398", "dsid_9d0d9dc7e1d84933b9f89376184b0142", "dsid_741377356f4f44b28aa59559207cf6a3", "dsid_0ea5ce154cb842ee969d8a89d03edd2d", "dsid_73fb2ce33e874e3aabf03cdd85de46be", "dsid_50432ac9155147a3983de651a2ca1c15", "dsid_be7e5cffb2704eb0b90de833ef6be6c1", "dsid_50cd7059170c4e2ea069bc2b9c423a61" ] }, { "question_id": "qst_0161::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2fb08b7571444bdf894607549642498f", "dsid_bd18e7244b064950ab6542ac457b5b66", "dsid_a1d8ea3adc68431ebd90cc40a51b5627", "dsid_82d3d60c2b2a47feab7cb07863897123", "dsid_8db16dd149b74972ae947d5525043b45", "dsid_1c53affffb4f477ca476bec3138b60eb", "dsid_5ac2be72b40041f5bd0c16dfaaac83e0", "dsid_add7fabe58f14c8fb51b10b4f4c1b6ac", "dsid_852961d22e0e48b194d7a3f8ccf17173", "dsid_5d5a67f7811c4b37ad26b2d389457618" ] }, { "question_id": "qst_0069::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_de9c1d389e964346a5d863550ba817a2", "dsid_acd812cda19c40fa8c5a9c044aacd6d6", "dsid_fa2f73378dc84458953df44e5e64493a", "dsid_033acb55079b4d42aac8ecc7d6b3b511", "dsid_52a32f2d0cda4647b7fa12dda76ed277", "dsid_4a8247d18bb5495e8b5c3476ffa5d4fa", "dsid_f227e95ee89440dc99f5b62ebbe5ddb5", "dsid_2a640cd6a0f44f07bd772d630f650e34", "dsid_f6ceb8f0b93a487aa66fac7778f85c15", "dsid_d81ca6c0b6104ad78c27a3b2cebecce5" ] }, { "question_id": "qst_0145::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_182c34ee62c54118b48c1390e77cd3ba", "dsid_e47c10f5903d46b2a1cd18a55b13f205", "dsid_9f1ef10c8f1240fba7b18068815b6bf2", "dsid_5d036e6cb6cd4ecd97291f6559ce497c", "dsid_398e6e782364403a9fcfc73cac68ac5c", "dsid_55f2e54002324abdacad31d19911bb86", "dsid_0934707f40ec4ee9bdafb705923018e8", "dsid_e6c579c3db744c318271bd1ca0ee3a96", "dsid_d7a07cecb3c2489c91df79f7989d8f39", "dsid_3dccd578f9ad4a2590556348012fc658" ] }, { "question_id": "qst_0439::completeness", "question_type": "completeness", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_821022eec9ff4aafbbbb9707ccbfd7c9", "dsid_8fe92e57ba9640bf8d2044a78e8f78d7", "dsid_21dafe53df4e4c5189ac389302dd915c", "dsid_1b6a17dd05f64e97abd48d43d7e49cdc", "dsid_1555399090284030afed1c0f50cd8097", "dsid_80b184a7b1f24b85b4a9380ed38926cc", "dsid_c3e029bc46704ac09a2e626e3653223c", "dsid_1b646ef282cf4c3ea3e665f637fe3fec", "dsid_ed968727168a411ca8628448b671c699", "dsid_3cd790e90a574f4fa156405bcbcb4037" ] }, { "question_id": "qst_0027::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e4b8a2a6b6594d77b1880327d506619d", "dsid_ff5780c299fa49d9969c3c52d5a708e4", "dsid_9dff54e0ef8e4710926767a81224d475", "dsid_8e92a64bc62547c0b4ffb5c150f82e29", "dsid_096e4752a7ed47e680ddc1fe7d3488dc", "dsid_cff29f8260dc414bbbbd9b00d3110e96", "dsid_a0f25cffcb264f0ea6eba5e501de0e08", "dsid_9eb677e48c454b16812ff65473c9dde6", "dsid_05837e0efe7c452dac714bca05ec9335", "dsid_a734f47b45294b98bf495b69d251ad8e" ] }, { "question_id": "qst_0418::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b837e1140d1a4c3d8efd2f42d9b6a799", "dsid_981f4a0281054f9aaaec7887156eebc4", "dsid_1a3a408d31b649cf8597fd1fefbc24fc", "dsid_32c5e7ca0e2e4eadac78b01a35cfed7a", "dsid_833b0d0146a94733bb7d9d1290ce85a8", "dsid_b885123ef2c04d508ee071328a4e5aa7", "dsid_d9d0ca701cc54d58adae135f69cc83d7", "dsid_e33c122264214846a8b7df7701d31482", "dsid_31638228bb754a38bf38d20c25b32bda", "dsid_2271ae6c393d409093b11b5a6d5b96ed" ] }, { "question_id": "qst_0024::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_74ebb73359364bc791707a92a3227e3a", "dsid_cf06e75be4f94156ac338f62e3ee028b", "dsid_00908a2dda4b4d359194a091019e8367", "dsid_546e110c14144ff9a1a02d9145c43177", "dsid_ffa7f082a77c415b9b56218a9933e04f", "dsid_8c92178c254d467c9854db67aac7c9be", "dsid_3692960f2ce1464bb19b76dff3203f7d", "dsid_9eda2ec9cbda4188936246d888df1d0b", "dsid_fcd7d1962d8049388cf57641652c7a04", "dsid_10ddfbb0b28b40c48848fa05804a3717" ] }, { "question_id": "qst_0339::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9cd367b225214f48a77dc58684edf6c3", "dsid_81208556437644a0a4dedf168baf81b5", "dsid_252a2d2d470e45f098eed81152aff284", "dsid_f2bf2583d4514059add358f439afca8f", "dsid_5085c9f4b4484f298dabd02c99d9ce03", "dsid_c69bea6ead23473faffcddf52ae34261", "dsid_22cf75c2a0ec4786ad7d32374a05cfaf", "dsid_d2e228e29d064a32afac866e030fb885", "dsid_6a43ab12dbe24c6daeb2d05c858e49c0", "dsid_92bf997794b046c892bf1f210b9aee09" ] }, { "question_id": "qst_0285::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a4c41bbd4875418dbd78aa08d37ee536", "dsid_f17df5d7a13f48028e27ea8c10d1c7a9", "dsid_ad1350d69d844e0894865eac8bdf4770", "dsid_33a9be3f84b647dd8fc211b7a3ef5b40", "dsid_821c4e1495ca41eb8a411225acac5e54", "dsid_184be937d34a412ab5e61366d54d8ed6", "dsid_b96245399ca248d294048e4cd65a200d", "dsid_595af30d9c194f8ea1a068a39dd9cedd", "dsid_4c46ea65a2654b94b21d690327c3727a", "dsid_d4cd5a360baa47969b8655c29ec902bb" ] }, { "question_id": "qst_0091::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9eda2ec9cbda4188936246d888df1d0b", "dsid_308e30551e644ada828f5285be203c12", "dsid_cf06e75be4f94156ac338f62e3ee028b", "dsid_546e110c14144ff9a1a02d9145c43177", "dsid_3ca0a233cf864428b9feb9c6a14fc026", "dsid_031284683a4b4e7fb0992705a9654095", "dsid_8c92178c254d467c9854db67aac7c9be", "dsid_180a51e1946f4dbda5bd0c67b6a972b2", "dsid_7a52693ff9514cb186208fc97b6cbffb", "dsid_8ae1ab7e41064f8a877100826aba3fcc" ] }, { "question_id": "qst_0075::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8c60fea2755245de913a78b2e29c617e", "dsid_5ea82211a864441c8f847c690182d761", "dsid_60b6b4cff8f440899bb213cbfb4e90ed", "dsid_3f7f32874e9b4ba49ff4fe28c0ef5de0", "dsid_08210ae89dc247edae7a470f03320eb4", "dsid_63ef1457c38c4aad9ad4c740996dcf81", "dsid_edd9e60a5f7e4f3eb0835259beab643b", "dsid_fb1c5aa67f664e7cb6f547e00245351e", "dsid_e780afc195644d2f9cc7bee7f9734a0d", "dsid_a1735d035c5a40138db9b4e8f4a00291" ] }, { "question_id": "qst_0470::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_aea4790d4bcc45859208d7d705f682a1", "dsid_eef6a2e56beb486fb0c0436372eeeec1", "dsid_ea6d8a686660485c8e570a8fcc96d598", "dsid_c9a1ca32dae14498a89c2ae1b1d5bfa8", "dsid_03bbbaa2fd64496395b8f6991372bcc4", "dsid_b6badcc8e17a42c8a151dface30872f7", "dsid_61519ce9a510484bbba40ffd6a7017be", "dsid_dfc235ef707e4b7a89c5803b90ab02a9", "dsid_71abc9ac8c374ee583cdc87594f01f17", "dsid_dbc2215b7bd44da99c726ef6ffe15714" ] }, { "question_id": "qst_0084::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3bc3170ead63470eaef9a53ca835cec0", "dsid_c875ada2f9164277aa9500daa4753c68", "dsid_98abe7c857dc40e29fcce7daee0e24a2", "dsid_46b8ac285de74cbc8d2ccb3d249a641a", "dsid_95612402ea2e4481aedc7c46cdbad170", "dsid_b91398cabd7648449afa9c62d4ade744", "dsid_48e13d429774409db3aaa152d19407b6", "dsid_9fb93cc3b2584cd887a487514f1b8ac6", "dsid_a827feb4bb854255b52f206161090f1d", "dsid_f26e949d7390411c93b24e77380fa76e" ] }, { "question_id": "qst_0043::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_795498b20e9c4d38883c8c46ab3336b4", "dsid_e4e5f7ec247b43efb0a19713cdc32c3b", "dsid_3f7f32874e9b4ba49ff4fe28c0ef5de0", "dsid_9f2e6aa7f6c14b1db1c6f922a1d4b4d7", "dsid_7e29cb5bf6ff4034b7e68370bb757d26", "dsid_2e36de0568db4dc2b695babaa01776cb", "dsid_50e057c2be0c4c2687226715198909f7", "dsid_826e96c94e60453d8294159b44e2f8d1", "dsid_5bb7fcfd6cb74a60aa3437781a30b1ba", "dsid_fd9be8a1d6a646d1b6edc32c9846b208" ] }, { "question_id": "qst_0422::conflicting_info", "question_type": "conflicting_info", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_44b11e967a5f4029aebb720891f66450", "dsid_6933f9241ef140da9d3bf98be52be867", "dsid_8733ed2132dc4232b2797c1d9a167cb9", "dsid_9cde32cd0645459b80ec513df27d1eb6", "dsid_1dceec2e5ce44dd989b690dc690ef9f2", "dsid_03042cec7a2344769bca5a624091092e", "dsid_941fd6f855884f418076ad5f36711433", "dsid_dd0fe40a2c71489793d566e46e95bb93", "dsid_385fd2388669469e87964b25d6428871" ] }, { "question_id": "qst_0094::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f7a0934d387d4b129077a8515e60801c", "dsid_869a1aa8c76a4a07934b712ff883302f", "dsid_523b276d1c584478b1a52a5cc52385c8", "dsid_5b597626db7e4604b31f97d2497ebfda", "dsid_bf7e79dc1331447598c4eebdc5d2897d", "dsid_7eef0ed2e4284e3ab3a3b74c17e95d87", "dsid_e0574ecced1640b69d8b8f0975e78bdc", "dsid_d9bd744a87db44378600cd2f2ffc6c3d", "dsid_9eb81841e34147c8a764f9e630c6fa41", "dsid_2544c903c31f446dab2b5d5c4a08c227" ] }, { "question_id": "qst_0081::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7a7c04eebb914edab49f734c8e814beb", "dsid_0d52203abc384b5e9399465ec08bafed", "dsid_11fffe94d564452da1c06eb998e18cd3", "dsid_75bc2477d775437fac0690975036c78e", "dsid_9294739602be40838570443ade196930", "dsid_59600ad815414e8c8d38f37d1a090bd4", "dsid_889a3085065142d5a21a708511ef55f9", "dsid_648b5bb829be40f8904844b3de8bf19c", "dsid_5a756a56040440eabbb9dd255c221fb3", "dsid_1a32ed9293a94784a2c317b7b79eab39" ] }, { "question_id": "qst_0068::metadata", "question_type": "metadata", "rank": 7, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.14285714285714285, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_58a34b025e1c4ccf8c97dbc9e042548b", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_e6f4d2efb6ab4fa0827a09b203e5e1e5", "dsid_3c69e5d0b7a6462d9c7d54f40dbb914d", "dsid_e19b6b7d1ae04f4287fda338801ed824", "dsid_51183030e764419caca09cfe647048e7", "dsid_d9a7de9e7cfc4fe0b7b57765b6d25ad9", "dsid_92798ef75b3f42529818d1f0acd636a2", "dsid_04d9f4e5eda14a2c98f3a97b9f40d64d", "dsid_e1c2c9f662f54663b984c170554c04f5" ] }, { "question_id": "qst_0107::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_509b34ee9a334f68a693e61dfbf558c4", "dsid_95612402ea2e4481aedc7c46cdbad170", "dsid_1faf80f5afa8490ea64c77c7cb2fdf8f", "dsid_37c04e51573745849480623d600f8131", "dsid_5e861a1f157a4cbe874637374a3c19b8", "dsid_bf7e0df04c8748458b5515c864c48609", "dsid_58c741f1b1a04fbf8efa8ff59d4b0977", "dsid_c8dfa383e9ef47cdb305bc0b9d3c5465", "dsid_ed13839354154887ba1c4d35de8fd8e7", "dsid_d79518c427654533a01173792f51f986" ] }, { "question_id": "qst_0079::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_1ff1ebf46f9b42d78f33dd7c283187ab", "dsid_48b40afb54b84af28b13cebf017c0616", "dsid_8e215f0af2f640eab127aa0fc0bf6023", "dsid_f3aecf9f7a5549fd8ae482a58c8e2161", "dsid_d295b7207fec4a6fb656f3e2a565d4e6", "dsid_8d893b30e5fb423fa614b44d456decd7", "dsid_06bc391be8a64a5388bb930455b7507c", "dsid_19e39dfd6b994dfe9c5f5c67da6d95cf", "dsid_1b6c76516224477ebdb142650bd01210", "dsid_ebd7217aa2174a67a3646fedeb72ebaa" ] }, { "question_id": "qst_0076::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b097ab5d9e4c41609f47432dfcb1508b", "dsid_c5b8c516b9a7423bb2db89bac98004ac", "dsid_19d5d2f6a4cc40cfaa6086ca39b2b741", "dsid_dde88b20cc4d467aba379b82caef518c", "dsid_9e1ea47f92604e56b3e146f0bbf413a0", "dsid_91edf1b824d04c9ba388cd66983fd795", "dsid_57719e17f05f4eb1a552f133def7a6ed", "dsid_7c96b7eefcee4b18852c0cc693cd527a", "dsid_e5040a0aa02c4af8a3b4681335b5e328", "dsid_37c24c697d9d462fad0e3afeeea9d8c0" ] }, { "question_id": "qst_0413::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6df52fdb96ae4edcb76464738bca3340", "dsid_845172d9dfc84fb1ac13457ca486f673", "dsid_249b208f80da4c18b4df5ed2004e64e5", "dsid_27f9e17069994d948a141196c64bbcfd", "dsid_452bb37502ac41a59f8b41f1cd79da65", "dsid_69d255424ad3485cbee6f836dfd7d70d", "dsid_329e7ecd6b304c30b61a30e07919e008", "dsid_c90bd544f3b5437fba711492db31aeea", "dsid_e42d7794db734f7fb69299207974d453", "dsid_6f86af4dec28405ba3cfbf6ed2b5c206" ] }, { "question_id": "qst_0363::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.8333333333333334, "retrieved_doc_ids": [ "dsid_b993fa19b3d44382b709e3107ab1ba77", "dsid_9a2f440e8f7e4801b76233767fcef945", "dsid_44b11e967a5f4029aebb720891f66450", "dsid_6933f9241ef140da9d3bf98be52be867", "dsid_1dceec2e5ce44dd989b690dc690ef9f2", "dsid_e68427c431e446bcaf25e4c24904a3ed", "dsid_e91e68b31a6f4935b87d6f74dd07e1f6", "dsid_2bb0a4d1958848b0939ed64cc2b911ff", "dsid_d94f119e2e5345b7852f4c0f63927155", "dsid_4c0d5d098d224e7495667823169b521e" ] }, { "question_id": "qst_0149::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_42c3e2d70229415d82cac2f9b795bf1d", "dsid_d3825457e10041d5bdff57cb8eea5085", "dsid_f869eb7c3a99487e977b8e63191e0227", "dsid_92f8264237c6461e89d6112d5321d1b1", "dsid_ead28230043e4f6da040e4d43ec19885", "dsid_9beb3249b34949ce85dee10d98353307", "dsid_a01e2c5bc383409ea30956af7d262849", "dsid_bc6b4096ed404ffe9d197041d258cdd7", "dsid_60cb31d4fea144b9b6764674c7aadbed", "dsid_fcb476561f404b7cace29600db6e42a8" ] }, { "question_id": "qst_0132::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d8a18c9f03de414faeacfda3ad53e315", "dsid_854a9ca8daf54716b75c22377df89654", "dsid_c40b65a75c4c46878b68994d5c68bfe2", "dsid_df7b19a443ec4bc481cd41ff7b5b296c", "dsid_140899f1c6644eccbd0efa6b090930bb", "dsid_60ac6e2164634ba48b8a62f296505d13", "dsid_8d740f99ecd94439b30a2bc46b6d52cd", "dsid_0ccf78108d9b44b9a55ec1a4196527f2", "dsid_ceff4fc0c1da46d7825bca62bda34df2", "dsid_4ce083d404fa42cbbdef587b2c9012be" ] }, { "question_id": "qst_0396::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f90b68cdda834c6fb4c76508d359c72f", "dsid_f940e7c95688406cb497b0b1dab44e99", "dsid_a5cc666385c242f2a37a57206cddace2", "dsid_2afb41d7ecac452a9952f4a9f987538f", "dsid_e7702842e5334127855135d300ced573", "dsid_da3315260ff94750ab53314700575d04", "dsid_55a0a2d2c5954728836fd941a053b5c5", "dsid_4b66d6d13e614ff89617822c0160e94b", "dsid_b1b99a34a6b44b74acc644d0da23bdac", "dsid_8e8f8a5113c3438e93575304f34aed91" ] }, { "question_id": "qst_0435::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ce60931a1aea428b8186228ee40bb497", "dsid_a134a3fd3c244a52a8d3e1ff515fdf58", "dsid_431b0713c42847bbac15f3354776db68", "dsid_03e22d519f9b40a5ac61a03e5c4620d3", "dsid_8116cb3e580543d7bde1c57df351a3f3", "dsid_7fc490f8522b486c8f4b665d5189066c", "dsid_5e6cd54cfe8745a7a2469c08d53e3a27", "dsid_f912a6d60ac647b5a463e981a66e8acd", "dsid_5555c6b607ef4785bf8df9d0970f743a", "dsid_108f8d34b1644e909607a15d23d3da1f" ] }, { "question_id": "qst_0082::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b4f5d647ef7c44389b4052782c959bbf", "dsid_0c504258cd0c432d898e6711fb90eaf6", "dsid_4f0b59f1c83640459247deabe08dbf23", "dsid_40831beb6d944ccab76c20d9a0d90a50", "dsid_5630a13942984b44ab4a6d8c9e286e00", "dsid_250f2523f3fa47f5b1ea994852d2ca2c", "dsid_270bb083ec0a4f6d8099198a1a44ac24", "dsid_c24658680cc94dcfb0a25ff8103ac916", "dsid_09cc299a9ff44277967e71c865589031", "dsid_a7a45616bf3c43e88fe913060b641b45" ] }, { "question_id": "qst_0083::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3b9483f629fd423d81ac49a7efff89c9", "dsid_f94eddd70c7e4a028e16c760809e9333", "dsid_ff1eebc9d68a47d0bae81e9ce296c718", "dsid_61f1b627af3d44469cd99ac1ab1ad195", "dsid_621365b3d9db4a239f75b58c6fcb82cd", "dsid_230f95f9f8df49dcb107a196f6a570c2", "dsid_e46195826bbc44d9861be37253de3ed2", "dsid_9a2d4b86d2b44f3ba2d97bca6e3a590f", "dsid_811b934e240d416e9bba821ecfa5217c", "dsid_849f7013dde4448aaedc6e8f6e5ea8d8" ] }, { "question_id": "qst_0037::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c1a652436b2e4c949f5de9f5da227cd7", "dsid_bb14e2f3925b4e27829c4f3c1aea3af1", "dsid_93e92c89d38048b595980f61bdd44fb4", "dsid_069cb1a12e64487d83a9372bbbccbbb8", "dsid_6b7484308b0248dc8b62b81639787925", "dsid_85b9079cabe94e279d698d3112743797", "dsid_adaebcf9535f47b5903bbaa5e48fd0e6", "dsid_e4c75621e0a342579bf249e227219d1b", "dsid_3c643d2e76d94a2594d87433bc7937ac", "dsid_d02dc8d4929d435e9ed5f2ae716f9697" ] }, { "question_id": "qst_0241::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_200c0d311412478dab2d58fc1957a761", "dsid_1ffa3f05e342408382daa57107ec4297", "dsid_459903ccf8854682baf3e0c468bba4e6", "dsid_d40b35f2a00a48e9839cd7ba1157e017", "dsid_ee4bbadb32f14a57a022161e7a3183eb", "dsid_8733ed2132dc4232b2797c1d9a167cb9", "dsid_a673ed031ece43c5bc4a0d27a284c8dc", "dsid_94616da0b07e4b47a5e26279e5f92e5f", "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_d94f119e2e5345b7852f4c0f63927155" ] }, { "question_id": "qst_0055::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d7aeb7de2b6c4ae88f46f4b0e4f2aacb", "dsid_582cf753660344acb9d061d282499d4b", "dsid_95c7aa5238f94570a8e0a8684736b666", "dsid_840703a1cc37438a84e9beb68029d80b", "dsid_0fbc0b5c391946058c8e1678e9e5e48d", "dsid_a3c54f368074465fb8efa091b24946e5", "dsid_5db9e7b87517429ca2ade855d6d5a976", "dsid_9b4942b103ed42f2b0c59a1e3e069557", "dsid_dc8153546fe941cd9b46bbd2b0a65b94", "dsid_e54f5abc4f0b4b5c869f16c986d64510" ] }, { "question_id": "qst_0282::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_a52a2354f2ef4a26be3e96dc2f9d6a63", "dsid_faf3246a6ce845e6b5df0ae52f439399", "dsid_821022eec9ff4aafbbbb9707ccbfd7c9", "dsid_ccd59f7c8e324eac950be6cbc1241a85", "dsid_1b646ef282cf4c3ea3e665f637fe3fec", "dsid_9872881b3dd64551961fa390fb7d4475", "dsid_7b2f4b37dfb642e19d78c5f9d2c43f16", "dsid_81323a96121047cf98e958463e33d80a", "dsid_c0c93abadf4c438189e23cb45256c6e9" ] }, { "question_id": "qst_0065::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7e82d2ac51114ca2b1c33ce55a96e499", "dsid_a2dc7d066f1a4b0cab825a3bd07d0fec", "dsid_3f7ef57a45d247f0b8fc4bb5cb30e78a", "dsid_821c4e1495ca41eb8a411225acac5e54", "dsid_9672fe504a9b4307bcd93d7309035c57", "dsid_33a9be3f84b647dd8fc211b7a3ef5b40", "dsid_821022eec9ff4aafbbbb9707ccbfd7c9", "dsid_b43d1efb469a4e3a86b3d8d59443305f", "dsid_a52a2354f2ef4a26be3e96dc2f9d6a63", "dsid_523cb04d67334d2c8992efbffa6af6ba" ] }, { "question_id": "qst_0036::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f94b93a4bc7948ddab45c371187dc6c6", "dsid_b4443ecc53154934902652d496c9d12e", "dsid_92bf997794b046c892bf1f210b9aee09", "dsid_1c60dc8fa03349cd8e67df7139ba91d5", "dsid_5776ac1f724b4f72a084d3e796f0af4a", "dsid_3ad691e18be849d897506f5276c24a75", "dsid_f5a0ffce1a574f28ae8656fd6d04ddb9", "dsid_15610c8da1264015961b9c5ece2e5dab", "dsid_e89a4013d34e443ab396d78e922f16ac", "dsid_8eaa2785974140b3894297324977a8ff" ] }, { "question_id": "qst_0297::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a3f94972391244f38ef773c05926af83", "dsid_9872881b3dd64551961fa390fb7d4475", "dsid_7d9067b4fd7c46b38a7d6eddc0e20337", "dsid_f6fa144362d7412c96d8d4798bda758a", "dsid_a8f1e01a4b3448f2b56ba353ee7514ca", "dsid_266d2445f4c044648146988664a98538", "dsid_c4f9a0d537a64e66ad66f0d687062368", "dsid_2b7a534e76c7405db71d7e770e31eb66", "dsid_0b1bb52f3c5d441baff768a188fb50a7", "dsid_92bf997794b046c892bf1f210b9aee09" ] }, { "question_id": "qst_0248::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_462f37885ea5406284727925ca5ded7d", "dsid_7c52f59a57b9405a9fd26c83d43bd2f9", "dsid_0a2cd37d53ff47d4aced289cd9a76fe8", "dsid_fcbaca2a322c4815bafc9ce000896338", "dsid_c2ddd5240eb04e16806d82103f7646dd", "dsid_aeaea63d311a4a1db8d96a17f804f682", "dsid_fc36d1d60e7e4b4abc7db84629563b7a", "dsid_3610da70e05a4cbb9c798e6a603cdb0a", "dsid_9d94736977e34b6ea6ddc53d021eabc8", "dsid_52ecf984d8584f1190958d63ba439cc0" ] }, { "question_id": "qst_0156::basic", "question_type": "basic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_32c5e7ca0e2e4eadac78b01a35cfed7a", "dsid_b4bd83b46612452e858bc02b380082e2", "dsid_914d75cb27ee4a06b3f69e445d715ab4", "dsid_f6fa144362d7412c96d8d4798bda758a", "dsid_94bc4dadf80d44158d27d1ef7d9e4def", "dsid_9c63777c789c4f7db9e391024dd887ae", "dsid_eaf5e08d11174eed9797914f9c361805", "dsid_7a9d6810cee2497f93113afa7feb0076", "dsid_c47239d68a2940338a4d0ed9cbe1fcfa", "dsid_2b7848a8491749f8b9b9a8c13f78445f" ] }, { "question_id": "qst_0096::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8138b96e181940418cc5067b2630064e", "dsid_562213b6a37e4912add20e642b9f9a15", "dsid_dced9d2508b54c45a65b20af24dc999f", "dsid_8692c99abe3d4ccdb158cf1abf37ae17", "dsid_429267fe79c74918a26272f6c0e48f10", "dsid_3ce244ef408e428bb409c59b283f3aca", "dsid_c5b8c516b9a7423bb2db89bac98004ac", "dsid_9e1ea47f92604e56b3e146f0bbf413a0", "dsid_19d5d2f6a4cc40cfaa6086ca39b2b741", "dsid_57719e17f05f4eb1a552f133def7a6ed" ] }, { "question_id": "qst_0224::semantic", "question_type": "semantic", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_88de52e795314a3ab7223f19aba8c920", "dsid_8a32317de3ee45d6a0042f54e842b9af", "dsid_808454fa642a4a5bb361b7ae4ead1352", "dsid_32f9c1a387554751b08d2ee34ea840ef", "dsid_ee4bbadb32f14a57a022161e7a3183eb", "dsid_4fde8c72c4e2491baea8654451a3a6da", "dsid_a673ed031ece43c5bc4a0d27a284c8dc", "dsid_96088536cbcc449eb676e94bf800486a", "dsid_acab3f167da743889845cd0cced08f45", "dsid_04441a6d15844c6f999eac76f6701367" ] }, { "question_id": "qst_0066::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8f5b801cebc34eb4b233bd0ef7c1b546", "dsid_d3cfe5f57dc14a9081fbed65d73dc736", "dsid_7f2a6f2d2de34f94a5b6f3d5d0d1f6a2", "dsid_0a549e0aac454f50be4ac0c34902276c", "dsid_8aecfa9775e74f61b9f53d3dc4362c50", "dsid_b38d68e2972646e2918fe199dae8ffe1", "dsid_f039509796f842cd84ba3bfb895c7132", "dsid_85a183c025574d05b5476634a6d67321", "dsid_147037e575b9478a92df1cb5b7643ad4", "dsid_b07abd6a88524179bb38ac4faa96218d" ] }, { "question_id": "qst_0042::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_28a0ac3c2505410eb3e0e84468a29f3f", "dsid_41af3c8f60724a4ba6210606cb4e3617", "dsid_096a9c7d5670442f98e3aab5dbe31684", "dsid_d11427550fd14a8fb8335ef28fca5176", "dsid_57a250dec9414edfaf0460f217f0700a", "dsid_929f3c93e285475f9198580a2cdc25ee", "dsid_61be163ae94f4fe1893a168640d73bf7", "dsid_ee9942076dfa4249a77338a401d25eab", "dsid_5af1ed23fd2a4bb3984ec368d0e3b23e", "dsid_2d7d6701e297486e80d23fee93603f03" ] }, { "question_id": "qst_0290::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1b7a5d6230f84eb0a38a97da0e0ce05c", "dsid_d079fa50bbdf40f38b3b85d108c2d1d7", "dsid_d3023727a4f14a66a67677202a889848", "dsid_7f944ea564cd41f495e212b0b08a7ba3", "dsid_8779ac72d6504759b701178632772f6c", "dsid_1e283d188dd349ddb2a72b3b64bbe5e6", "dsid_5139b75526bc4cb0a55b63eaecebe105", "dsid_b8ca1ba335b44d3590f876085e550916", "dsid_b78dea171a724bdcadbe6b0547ef106c", "dsid_ac872c34184d49c49fc700dac6e12bf8" ] }, { "question_id": "qst_0042::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_eea41d353de049aa89be98b6f9cf5de1", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_5d161fbc61f94eaeb6098a9f9275d38f", "dsid_58db062ceb7841ec94eae202f9a244ab", "dsid_cbbacfef259d4c69bced199807feb1b4", "dsid_9dc49b7a589c4ff2af437f1600f283cb", "dsid_20072897c77f467fa7e455b06ac8469a", "dsid_329445a9793c47168c67a6241e02fe80", "dsid_93faa7669dd9448d9015ad6d5879c050", "dsid_60fb951d61f44fd49fdd80daa2013ffb" ] }, { "question_id": "qst_0458::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8cb8070d41604b67901e60da07713d92", "dsid_bb14e2f3925b4e27829c4f3c1aea3af1", "dsid_68d1cb3084b548e886098e653070e99c", "dsid_a5d293df3be843caa748deb7c0946abf", "dsid_f151ee037d5a4f538e410ade668e6f20", "dsid_3c643d2e76d94a2594d87433bc7937ac", "dsid_5fc2dba9f6ac4af2b49b4f546a4298d0", "dsid_f0567bbb5dbd4a52aef951e11ec13f2b", "dsid_935850575a6d4c57b63afb39087b228a", "dsid_c5fef1e14e974f589aa495b27a5fd0ae" ] }, { "question_id": "qst_0242::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f23926802cf44ae4b85c4410682eab90", "dsid_3c18e45fbc064bdeb056d94d621846f2", "dsid_521ceaa63b844d85940cd637e799da97", "dsid_5e861a1f157a4cbe874637374a3c19b8", "dsid_e7dd8aa445994a6382c0ebeac6151bef", "dsid_96ef90155be7446b9629e7dcef4cd768", "dsid_b3fda4a46a40497083b15c09770a34dc", "dsid_a5704272fc6b4040b66fbdda9d1e1d05", "dsid_4f584c93c8b54e4b954238ca2c80b956", "dsid_d7a07cecb3c2489c91df79f7989d8f39" ] }, { "question_id": "qst_0347::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b8691ba2e29542838afae93928ad816a", "dsid_ea2d1b2252a0476fb8351fe49038ec39", "dsid_dab17ee5c3924a9998b17b3591bbae28", "dsid_ecf32f68ff2049dcbaec9294337da274", "dsid_c3be99dbca144db488c037dd2f0831ef", "dsid_c5848fca4e1c4cc48164aa3e53e5741b", "dsid_9ed01291d81947f49500df5fb28ab724", "dsid_10a589a5ee9048d9a33deb4156037171", "dsid_deb0a67e0e55442f9ab5a3bc41bdae68", "dsid_aa97b7293f9f4f3c8180f645e4fe5911" ] }, { "question_id": "qst_0200::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_5e38ed781dfd4b29ad70d79bd9918040", "dsid_d052608691b14b6c98f8047aeb4a4b75", "dsid_565f801f14bd414799804ab3f374cd46", "dsid_8aea1b4f1fb74eb0b9baee6038486358", "dsid_fcbe18dc446e41a48bcc2d7df18ef9c3", "dsid_72acbfa2791742cbaa22e34bacaee53a", "dsid_e37e7b7723e34b8087ef051804bfe85e", "dsid_f4e0ec9fc27c4e52a55b34af90fcd1b0", "dsid_857e8bf5ab2b423e9bc8bd5f6ea1583c", "dsid_e17cd9b5aaba4450b7e07bc36c0e7e3f" ] }, { "question_id": "qst_0083::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0abbd8a3a39e4cfda70dcc403b22f64c", "dsid_2384c6217a874538986694d017a44f8d", "dsid_78fcc2207a7a41e0baa0025286e243b5", "dsid_2d7d6701e297486e80d23fee93603f03", "dsid_3b5f7f9794cf4f87baa5e1b1d582ded9", "dsid_1725baa1dbf24a938e8acecded3dba3d", "dsid_a4abf484b679498096d46b21d91b523f", "dsid_0a63462d31014293bf92e8ef8a0d1d58", "dsid_47fa00c889fb4446b0481efea384a7ce", "dsid_3594e7b29c724c95a70ade7d13ff17d1" ] }, { "question_id": "qst_0141::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_acc33ff059404020b0ff26e19ad2b107", "dsid_5e38ed781dfd4b29ad70d79bd9918040", "dsid_85a183c025574d05b5476634a6d67321", "dsid_c89bd8ceac1b453f8634e570b4c019d5", "dsid_5c40eb781f214e338ac2ac183d0115d5", "dsid_8dd74955eb4440c0b573eb53aed54d6a", "dsid_1860c46106e841b986efadadcd8370f3", "dsid_b607b02d987f4e53853d3afead88de64", "dsid_d8b0c3c09ef2411f8df35e449e91664b", "dsid_65ae7369bc414e12920ddd8772328f5b" ] }, { "question_id": "qst_0445::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_cb0d800639e84d68ae035596c7f6f1f8", "dsid_2ad75f27c36b492f847d5b9c63492e4b", "dsid_2ce9ae40d5d347c1add2b31c4d5a8c3c", "dsid_7b972e9b7ac847bba3f4145f56970301", "dsid_32c5e7ca0e2e4eadac78b01a35cfed7a", "dsid_2b7848a8491749f8b9b9a8c13f78445f", "dsid_29a1ac52383b4537b8f51186cd151401", "dsid_89889daf6b89426ca90049f313d4324f", "dsid_0018a3b3fa8449298cd96ce4ce83538c", "dsid_319db001cae246bbbb16f441b64d9b82" ] }, { "question_id": "qst_0371::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fc356e32e9a64c3199be174cb91432eb", "dsid_a1d8ea3adc68431ebd90cc40a51b5627", "dsid_3ad691e18be849d897506f5276c24a75", "dsid_70f3865fb0b84b7e879497718c0dd027", "dsid_bd18e7244b064950ab6542ac457b5b66", "dsid_37f8be153a5c4aefa03531fc32f9cf65", "dsid_8db16dd149b74972ae947d5525043b45", "dsid_95cc7ee37b134f1a84b86fcdd14922f1", "dsid_305fa5509d34415d924578012fe1fae7", "dsid_2fb08b7571444bdf894607549642498f" ] }, { "question_id": "qst_0277::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_d9a7de9e7cfc4fe0b7b57765b6d25ad9", "dsid_715031e81f3f436fa0282fd0f248a716", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_58a34b025e1c4ccf8c97dbc9e042548b", "dsid_17b2c4684dd94f7cb5bab6843377fa9b", "dsid_e6f4d2efb6ab4fa0827a09b203e5e1e5", "dsid_e19b6b7d1ae04f4287fda338801ed824", "dsid_3c69e5d0b7a6462d9c7d54f40dbb914d", "dsid_92798ef75b3f42529818d1f0acd636a2" ] }, { "question_id": "qst_0322::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_bab6aba4c06e41e9ae46b9355f644a7a", "dsid_dd34a1f6919e456ead57b0c3adbf4d18", "dsid_2619d808a84f469e96acb6934752a177", "dsid_c67e1098ee1e4d438a041539a79bfc15", "dsid_ab6663f3852d42789dbff6171d1aa21a", "dsid_6f0be39ff1ae49c19ce0662bdd6de352", "dsid_e2f5045f10f548579ef7a3d822e01fc7", "dsid_221b5f5a9986471193fa94fb9ceb95a1", "dsid_1415c2463492415693a3d4758847630b", "dsid_7a0a53e75ea84e60b0b438c9a49e8686" ] }, { "question_id": "qst_0109::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c9e70cb6eee1471baba213a976a8dd05", "dsid_47cc8ab990f2448c947c433ce8a601b7", "dsid_7bd84fafdace42e895aaca4914cb400c", "dsid_fff0eaf808b149ae93f73d4d56dc2dac", "dsid_d05799c9845d4f1ab3f1bae4d6cdf747", "dsid_e6f4d2efb6ab4fa0827a09b203e5e1e5", "dsid_85a183c025574d05b5476634a6d67321", "dsid_d8b0c3c09ef2411f8df35e449e91664b", "dsid_5ef618fee40741bb9fac512e2448e560", "dsid_c78821808b834f94a512762f8d198ecb" ] }, { "question_id": "qst_0294::semantic", "question_type": "semantic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_741e17f34f654ee7b1fdea7eef31af8e", "dsid_702bc520c52040428799fed41c63c8ca", "dsid_d92de412c7494a15af914db5b1c9dcae", "dsid_77351104a57140ac8e065d0a6bcf2046", "dsid_2f36a22850204ef985a6b908dcddd90f", "dsid_a55032c4c2064671a4b0abf975438e1a", "dsid_a9bb70a65894469ca2cc97a8f9dfd7e6", "dsid_46a4cb87db414e769f2df86f01626948", "dsid_48fa74f556aa4f0c9996b53170693e8f", "dsid_eebd7fedbdf042b984e26457aacda3f3" ] }, { "question_id": "qst_0284::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_320b9c2050b44a9194ebc681a9cd24e1", "dsid_8cbee6fceef345118f2a8e90d862226b", "dsid_497e2620f4ab45f29db9aa4193cf9e2b", "dsid_3610da70e05a4cbb9c798e6a603cdb0a", "dsid_eb9ec0aa38fa4624a866b75567f79d3a", "dsid_0ec0664a173d449f8391e5d4b0ce66de", "dsid_a8a928a11b814382b9ae9bb1fd5fb0a7", "dsid_be8f78ce442a44018497148aacbf99a4", "dsid_4ba88b8d61634a348e08086a54741f92", "dsid_2c9dcc3afa3249f39c725884e5be5c72" ] }, { "question_id": "qst_0051::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b7e285fa5700488fb87461b56db5bec7", "dsid_f8c8a11bcb224dc0a68dcabb8aeb2a38", "dsid_ce646062847e440fa50c9838f04f73c3", "dsid_6b7ff0ba89ed4a589afacc660abe28bd", "dsid_e4d51a846d3741b98fbb652db9e6d1e0", "dsid_0096a5f5ec3f40fcadc2b316bd4a2933", "dsid_308b0d2adccc4f0f8f27ff8a782adcc3", "dsid_9b44585b47884906bec2e2afeab9fcc5", "dsid_178f5ada27f64226843c83f032cf3481", "dsid_750cb12e4aa64edea40f7ad0370d37ef" ] }, { "question_id": "qst_0016::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3f7f32874e9b4ba49ff4fe28c0ef5de0", "dsid_5bb8a925b2414de78011566238f8e4a8", "dsid_10ddfbb0b28b40c48848fa05804a3717", "dsid_d7f0edbaa28147da807b5b85ea495bd1", "dsid_bc83adc1e9cb4b739128a6b574a6fabe", "dsid_329e7ecd6b304c30b61a30e07919e008", "dsid_be4b992c9bd941c5942e600ef769f6e4", "dsid_acab3f167da743889845cd0cced08f45", "dsid_005f7a937cad4b3cbb30d9d93199e22a", "dsid_826e96c94e60453d8294159b44e2f8d1" ] }, { "question_id": "qst_0259::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fc009de77554459b89e6b4340e8eca82", "dsid_662978747ddb47338b62f7d87c391ca2", "dsid_168fb762b10c4b77896d7821e4328094", "dsid_b8691ba2e29542838afae93928ad816a", "dsid_1b6c76516224477ebdb142650bd01210", "dsid_8f78680229eb4c1581d99c16760ab528", "dsid_e2327ad35ceb466489dad854770739bf", "dsid_f94f087fa70c4bdcad3b6d124c7c0d3c", "dsid_323f0a727b2548478b6fb4e3d4bcf5c9", "dsid_78555442db264e8bb41349f7ce09e747" ] }, { "question_id": "qst_0203::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d94f119e2e5345b7852f4c0f63927155", "dsid_43f8bb67a970429aa5ab5d3a7cbf4d4f", "dsid_3d745c88c7e4426c9548350824f0d9b8", "dsid_bf9b1bfc708b430ab18eac8119712de0", "dsid_5edbb8466e9e4edab981f94adace64d0", "dsid_ff0219b98a3c438dac62c06080c6449f", "dsid_c32fb8db89214c92a250cdb31478c12a", "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_8b58fca07ca34699bb2cce932e3dd24d", "dsid_50e057c2be0c4c2687226715198909f7" ] }, { "question_id": "qst_0408::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_e190b0de0d27466fa91bfc18b0815311", "dsid_378ac277dfa744c8acd16b30af9e364e", "dsid_190d9732b59b411d8a6bcc59fd61b309", "dsid_5147667611a54584aa272d1469d8f6a7", "dsid_7e5744fb69c044fd92ed0a6e85d5cac2", "dsid_96070e9bccf74b43bcde8e0e934d2428", "dsid_1732e93e187f4b5eb6352e1ae9eff5fe", "dsid_2dab968306594149a14ef27b2a1641d1", "dsid_1408617f31a94530899809f9961898f1", "dsid_6807c651b55546559733c7c6f8dc133d" ] }, { "question_id": "qst_0126::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_310b703bb5b04dc7ba499dfab083c8e2", "dsid_ec1b28cb66f4415d9c774ed43fc54fe6", "dsid_912991a0a2ce4000a55b1f67bda8c3a3", "dsid_d7d30fc8197d4a8eb3f968bf527af14a", "dsid_d289e6865ca840ff9f9f31425d7f146c", "dsid_0b2e9d92999d48f3b3029b1c798348ae", "dsid_873e4ae38e9d462099dd18f834ec4da7", "dsid_9f5e6a738afd4039926b2036753bd934", "dsid_2ae872fe39bb4b278c306edf1d07a6b6", "dsid_c9e70cb6eee1471baba213a976a8dd05" ] }, { "question_id": "qst_0271::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_db87c0dec8444b1d83b06cbbe725d3a8", "dsid_638bdb9278184f4f873b02c6e55af122", "dsid_463573b2edfe492ca27878c2ef9bc640", "dsid_76bff9162fe24370bbd2f3ad63f2ee61", "dsid_3d1b7e13cb98482a98efe6833cf44d85", "dsid_04e939350c834d1186e63048e4c5681e", "dsid_5bb7fcfd6cb74a60aa3437781a30b1ba", "dsid_a49b2f5e6fee49b998b5188b80885c42", "dsid_c875ada2f9164277aa9500daa4753c68", "dsid_408c94bb372743f4a622da1d30717c83" ] }, { "question_id": "qst_0135::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_63a1c6c12e1e4d9cae67056673633192", "dsid_ad3961a5bbd846d09d101bfa951ab739", "dsid_903a56e9690a4409aaf19c573262e1f4", "dsid_5e25891b15a84650957ad74cd11f0c85", "dsid_c341da4d9b0947cdb1077f66be06e901", "dsid_eb0568617e2a46bfa45a9cd4b49acce9", "dsid_bc31f46fadd749ee85beea1579a4f840", "dsid_298465b56d664ab9b22678d74630c484", "dsid_2aafde11ecc9471c985ff58e0c8cd1c1", "dsid_d934f97d888a495da2c5f9375850bdc5" ] }, { "question_id": "qst_0065::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1eba87b0b11b4f31a006f0cf164b8509", "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_51895b7e9f2942898291b66b72886b15", "dsid_78001368bf0e456ea8a4d0417b2c5b01", "dsid_9bf6777d45b0433b947efa6bee592d81", "dsid_f5a0ffce1a574f28ae8656fd6d04ddb9", "dsid_15a9d93cc59c4241a97a7e72f749a216", "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_f192d471aeeb4dc3945e49f35dd6183f", "dsid_4700682214424a7aafe594325c48459b" ] }, { "question_id": "qst_0050::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_089917448e654911bd186650ed12feb9", "dsid_be8d2802cf3245088a7e7f861f317b31", "dsid_e8145cdf3b68404fa7c74145573931ed", "dsid_3e7cf5be42d84f2880858c726b65998f", "dsid_353037e855ba482b9e4ec4d4102c51d7", "dsid_8d06e6c45e744197b6fbbe7c5792f094", "dsid_659fedb51e00463d8baaabaf9193cc35", "dsid_7f9e5e1c40184e0b8ae1040e9ea3692d", "dsid_27c5da43b81146548605041fcffa331d", "dsid_2102faf47399467cb01ff8dc36a1bc34" ] }, { "question_id": "qst_0157::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ccaf626687b940f3a7309e09c1ed5c1b", "dsid_e15750196c7643729e122d2e1cd7fdd0", "dsid_7fefb56c1ba24ab99d25ed49e8f76847", "dsid_84742ad3d14d49b2b14e9ee51592ba33", "dsid_f94eddd70c7e4a028e16c760809e9333", "dsid_621365b3d9db4a239f75b58c6fcb82cd", "dsid_0553f750bf4a425ea7e53c353cfc58ec", "dsid_61f1b627af3d44469cd99ac1ab1ad195", "dsid_43dbaa6340ba4f6b8f5bea0fe2354bc7", "dsid_3b9483f629fd423d81ac49a7efff89c9" ] }, { "question_id": "qst_0240::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5402831508734a51adf649603be4b075", "dsid_c0c93abadf4c438189e23cb45256c6e9", "dsid_595af30d9c194f8ea1a068a39dd9cedd", "dsid_205c62f1ace8402abf9820662d82ccef", "dsid_096e4752a7ed47e680ddc1fe7d3488dc", "dsid_5cf1a6657f1848968b2fd154066950ae", "dsid_d64d56373a0e48d1a5106087f792480a", "dsid_2c5a4920775f4b25a9d83496eff55bc9", "dsid_557c47d8aae543afa40b6109b257f863", "dsid_a1735d035c5a40138db9b4e8f4a00291" ] }, { "question_id": "qst_0094::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_bec90d2205534ca9a016f244e13884e5", "dsid_ccd59f7c8e324eac950be6cbc1241a85", "dsid_ec406aef8d0342c78ae27025670daedf", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_f3ef2d43e92a441dac7ce47ad319853d", "dsid_c40d07ce7b5d40ad847f3f952432ee82", "dsid_5b3f9d2ec2374a7f8146c10cded208e9", "dsid_10fc70076336485483c088238c7d144e", "dsid_91d986a3787b43b7929588416e5a950e", "dsid_f00e7cea721c48668e5805db61204124" ] }, { "question_id": "qst_0080::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ad3961a5bbd846d09d101bfa951ab739", "dsid_b135486896b5442bab43e66e6a4191c4", "dsid_1fa5a44034cf4fe39cdcd786e099651c", "dsid_903a56e9690a4409aaf19c573262e1f4", "dsid_699d8ef4cf5749b38328b0b15a53552a", "dsid_63a1c6c12e1e4d9cae67056673633192", "dsid_bc31f46fadd749ee85beea1579a4f840", "dsid_08a5ff6ebb3048b49b54576b304909aa", "dsid_216d7ff041d54b5dbbbf2e4229eaec6f", "dsid_eb0568617e2a46bfa45a9cd4b49acce9" ] }, { "question_id": "qst_0404::constrained", "question_type": "constrained", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_5b2aa3e1e8b54705875b9996322d762e", "dsid_adf6c58b4c984788b5c4de7721e33346", "dsid_9df81e849ade4badbb49776a176bd4cb", "dsid_d844ac57e2d54b6fb74cfcf91f29cbb4", "dsid_66671ce833614efb98d15a531bdacbb1", "dsid_337be594f53c4fe0bd9cc302d0b9a6e5", "dsid_3faa795a435f45b8ab16fa41c2f354c2", "dsid_bf5bb789b1434b06a3999648348ec86c", "dsid_08a5ff6ebb3048b49b54576b304909aa", "dsid_59d353fe72124e21820816f628e7bab7" ] }, { "question_id": "qst_0164::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5c05d2a74c484906852925d1cf1daa70", "dsid_0a2cd37d53ff47d4aced289cd9a76fe8", "dsid_60b6b4cff8f440899bb213cbfb4e90ed", "dsid_6a652323823643d4b5086e582e28e495", "dsid_9d94736977e34b6ea6ddc53d021eabc8", "dsid_ffa7f082a77c415b9b56218a9933e04f", "dsid_7359252dd4a647d4b51d51e098d015d1", "dsid_d34152a13c7a4def864a13d9b1e2c989", "dsid_be4b992c9bd941c5942e600ef769f6e4", "dsid_2449d374c39341f198e49addea82576c" ] }, { "question_id": "qst_0312::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0646657fd7e3448faf959a6883257e23", "dsid_71c7f98379214e948803191d23f164cb", "dsid_c054271735b541ba91e42c4dbe8a4883", "dsid_1fb4fdcf54ef4ec9a6fdc9081f7c9dd4", "dsid_aaaa7e2f91244354b3e63c8ebf1365a2", "dsid_fbcdb6891aea404da7f2f5b40163a294", "dsid_7d0507136883475a9300c3bc533fd423", "dsid_efb6f3e5619548f1bb44cf043643afb2", "dsid_b9d80919bc344a98ab14145e11f46ff3", "dsid_57a250dec9414edfaf0460f217f0700a" ] }, { "question_id": "qst_0215::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a56c6ba5a2094d8b83c6c70d025d940e", "dsid_d7b6b915ba6a47d98fab7dddcc43e50c", "dsid_140899f1c6644eccbd0efa6b090930bb", "dsid_40906807c7d3474eaf4697a3926a77ce", "dsid_ceff4fc0c1da46d7825bca62bda34df2", "dsid_8d740f99ecd94439b30a2bc46b6d52cd", "dsid_c7bcf2745c8f44228768c7650ed1358f", "dsid_40af385d26b44d71820f051fb8915b2a", "dsid_ea09bd3d4aab472a82c4825d7413c844", "dsid_7afe5f98d6e64cc5bbddc0291f0afea6" ] }, { "question_id": "qst_0265::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_876b1a31bcc7409ab560b9ccbe5a0d41", "dsid_bad6bd97d4db46a59cb15721b6537cd8", "dsid_4ecb42b192a1421885f6fbe8a7cbde8b", "dsid_4a74904e96de469eb6959ac78443f035", "dsid_330be20dd4224d8dad24eb633121ba06", "dsid_7643f1711a4546f3a01ac9dc1726b83f", "dsid_a9d291a78fc749758e972678d5f667e6", "dsid_3638bd02227d4882ad0605f1e8b767c0", "dsid_7e5744fb69c044fd92ed0a6e85d5cac2", "dsid_58bdb9f3bb4241e089f4f2b102765db4" ] }, { "question_id": "qst_0139::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3b5f7f9794cf4f87baa5e1b1d582ded9", "dsid_9f92a42dc8d24c93a0a696a0727063b9", "dsid_2e7c231eded142fd98385b9c3b2e6f04", "dsid_fcbe18dc446e41a48bcc2d7df18ef9c3", "dsid_feb4b72c7864456880a4be89ea166d38", "dsid_5993742b73f744c48d178c6151706d58", "dsid_0896c7487dfd4088b71bb1f1b4304081", "dsid_f9c5df417f65438f8755bc446ead4b65", "dsid_ee73e11790f545c295f4f68acc8df5d7", "dsid_37a4a9019f4445049ec551fc3dbae49a" ] }, { "question_id": "qst_0369::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_e7d8c585e8594fe481975515246d04f2", "dsid_31be9c3c708a4a5797d50a579c137587", "dsid_a560a94faf9c40ec9782d125d94fc009", "dsid_6e528fa402c34043897649be6aa6df3d", "dsid_1e7f42e5c50e49a69f8ec1a1d8e2dec2", "dsid_ca42a8f18d784c879c5a6fcfe3a484c0", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_fe6c423c71f04547b10f617e4a01af33", "dsid_5776ac1f724b4f72a084d3e796f0af4a", "dsid_d32eb5051b554e038222c3bf399acef8" ] }, { "question_id": "qst_0403::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a98ada14af2142a7a46266584d05c846", "dsid_c8a4367928d447d0bc86535046bd2a04", "dsid_91d7f359508c4476a6f717ea2b33aecc", "dsid_e4b87295de784676aece27d99728260b", "dsid_f226b067355f408ab8c6f59f3bf8a24f", "dsid_ceff4fc0c1da46d7825bca62bda34df2", "dsid_ade2c29c80a44e9eb74cd038f401ea1c", "dsid_e5d02ab7a7bd4244996ddc39cf76db35", "dsid_8d740f99ecd94439b30a2bc46b6d52cd", "dsid_bcbe06121660466f86fd22051f7dbb6d" ] }, { "question_id": "qst_0194::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0c8ea3b2a7fd4392b1f480fe5f0c4725", "dsid_c14d0841d94c4c07affce6d0d61963ad", "dsid_b27f56e2a56e43d6a7429585549ec383", "dsid_078a6387b87a46b1b782c2f89b3235df", "dsid_bd7caad8adb2405eaa7ad78c4ffb59d7", "dsid_e98f49df839f4283be1a9e336f7eb5d1", "dsid_37c24c697d9d462fad0e3afeeea9d8c0", "dsid_7b664a35119744e9be84ef1f403e7494", "dsid_9a088141571b435f995568199ec4bcc9", "dsid_19d5d2f6a4cc40cfaa6086ca39b2b741" ] }, { "question_id": "qst_0214::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_303196c831af4d1bb38a1e2546228a09", "dsid_53046df6de1d411aa88c6ecb288d3dc0", "dsid_5e65a1cb22b6464e9f36d0fa2d1f4317", "dsid_49bfa865c7e4423081811beed4f09bc7", "dsid_f79c940cf70647daa6cf1c62de8f1a7f", "dsid_8657f98ca70d4f26989a2d0776b13145", "dsid_f6e3b7ad413142019dbb5a5fed07f548", "dsid_3ad205aacc4147f585eb22aaa7f9daa6", "dsid_24b534c518dc4f1898e9c0ded6265261", "dsid_d9984be4f79f4f3587b02b29635fefc9" ] }, { "question_id": "qst_0018::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1f81f208f51a4aacaca226109bbedc72", "dsid_09f0c8ab37c44ad18a24d080835f25c7", "dsid_a67fe17a8dfb4b71b074352afe3fd471", "dsid_971936ac97434f8ca583dae38f229345", "dsid_f877499fe0e946199ceb2486f0ad5ec3", "dsid_6b60617d32f84e8384a25deaddf51de2", "dsid_42468da4f64040cd83a6423153c3c190", "dsid_9df81e849ade4badbb49776a176bd4cb", "dsid_6d096d1b51d046c295c028e328c97a82", "dsid_fc7478eb943b40aaa3aa80f7a0545cdd" ] }, { "question_id": "qst_0175::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f02a338c2dd546b3b025b5d5d429ad4e", "dsid_c27483a128094c9c8cef4eef00074763", "dsid_34120e1690ff469da2c3b043db4827e9", "dsid_e36cca24c80e4b50994878eb1c66fce5", "dsid_5204d2a9973a49bbab1ea7a9fb129503", "dsid_0e50cf59248d4a62bbd399ccf8f52c32", "dsid_f4e73684a305499e8a4797f873c9cb4b", "dsid_cc0f1f6ef452440d8d22000e962682f7", "dsid_fdc30024e874443eb02ce1fe78bde28f", "dsid_1867cdf06a00437194b3250c75321a92" ] }, { "question_id": "qst_0160::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c84c8581594e47d49e82d024a6da915d", "dsid_d17f040352fc4b78bde0e93c6c4a3e4d", "dsid_eb6986a727f64742b70b58e928150593", "dsid_73a6ec6ec5964a408ad2c51fd93437d5", "dsid_572a82a8b16949be8c5445f60fadd851", "dsid_a116143e0b5149a7b3b00b27fc30e6ca", "dsid_56ae493e39e443928013e4f009b3f815", "dsid_4ae71d2f20b4484bac7c246c3c7fc0b2", "dsid_82122f8103524e75972fc64ffde2c6ac", "dsid_672bd385942a466798c7764d484e10d8" ] }, { "question_id": "qst_0063::metadata", "question_type": "metadata", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c1f99c50101c486bba6926609637b94d", "dsid_15dd15bfbd0f42c485a9b5b9ace6a3d1", "dsid_09e8f39e4ac843f0b6acd211afe15b9f", "dsid_48e13d429774409db3aaa152d19407b6", "dsid_4977201555ed4d2487b0faa95d3fb8bb", "dsid_b91398cabd7648449afa9c62d4ade744", "dsid_b0348986b49e44ffa6d4f8f8ce8abe78", "dsid_5a2de6708f26400fa102a69910fd49bb", "dsid_3bc3170ead63470eaef9a53ca835cec0", "dsid_c976027b41f24d82a52c2a6411f6b444" ] }, { "question_id": "qst_0062::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_702bc520c52040428799fed41c63c8ca", "dsid_ee9a0f08dda843a281e1c362f93daa4b", "dsid_1dc182f68d6642cabbf055248ed2f1a4", "dsid_1725baa1dbf24a938e8acecded3dba3d", "dsid_ec0046f324144e048a8ce93d5e03e099", "dsid_9d71aaf572934422833f49793215fba1", "dsid_dc02d79d76aa4aada99a7a136cc15253", "dsid_6dc5b0ae2d064494b32013c8aab250ec", "dsid_1732e93e187f4b5eb6352e1ae9eff5fe", "dsid_d92de412c7494a15af914db5b1c9dcae" ] }, { "question_id": "qst_0096::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ca09c9594d0f4293b971c96fa9ed647c", "dsid_86783f1b3d154ac7b57c6d3d71bb5efc", "dsid_380aac4dd4254d44a2b72d253fe0e268", "dsid_21486610d7314d84980518cb355b6a97", "dsid_d329f1ceb55047409a0c3c90bdb03ab7", "dsid_24bf9d27747a4b4290ca85a6c76c1660", "dsid_51895b7e9f2942898291b66b72886b15", "dsid_d8da02f9a4f742d68f99b87c5bedac89", "dsid_1fd146ee26fb4d15987b5a94287927fd", "dsid_d9d0ca701cc54d58adae135f69cc83d7" ] }, { "question_id": "qst_0082::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_4c1c5fb53ca2432995ea42adc4330fca", "dsid_45638a5688314bcb969154b2a9dcbe13", "dsid_8eb8e41a45bd42769f79f85ed8d3ac82", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_0a549e0aac454f50be4ac0c34902276c", "dsid_f5b3f50f2bd64e828c78ee57a89bef55", "dsid_e19b6b7d1ae04f4287fda338801ed824", "dsid_75fff671684246fc80766d44866bf248", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_a8d4589e95974539a46a5907769a06f3" ] }, { "question_id": "qst_0010::metadata", "question_type": "metadata", "rank": 9, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.1111111111111111, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_be286e63a11445edaca7630db846442d", "dsid_b3bb79929639401798f43221b1671498", "dsid_f30a7c8acd944954bb58f1243da983d6", "dsid_90cb3306a4b143cd8c5f7daf061022bb", "dsid_eeb1ffe01b8b4631a251110976f80c92", "dsid_ebaf66cc8e964faca581f4af7a8095c5", "dsid_521ceaa63b844d85940cd637e799da97", "dsid_9bebce3819284bfe84ef3c1c1582db92", "dsid_1415c2463492415693a3d4758847630b", "dsid_7eef0ed2e4284e3ab3a3b74c17e95d87" ] }, { "question_id": "qst_0391::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_0488c203b7094b0d9f70c13891c7bef9", "dsid_c8f4a68d2cf248a7b941963c8e52d7df", "dsid_4700682214424a7aafe594325c48459b", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_b0ab6348d33c4b6787fff72b683bdae4", "dsid_ccd59f7c8e324eac950be6cbc1241a85", "dsid_15a9d93cc59c4241a97a7e72f749a216", "dsid_9eb677e48c454b16812ff65473c9dde6", "dsid_5601734700e044fa9f24731e6f09e1cb" ] }, { "question_id": "qst_0288::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c2ddd5240eb04e16806d82103f7646dd", "dsid_aeaea63d311a4a1db8d96a17f804f682", "dsid_7c52f59a57b9405a9fd26c83d43bd2f9", "dsid_412ac02bfc4048d6b78b9c42aaeaeeff", "dsid_462f37885ea5406284727925ca5ded7d", "dsid_fcbaca2a322c4815bafc9ce000896338", "dsid_704e14c174bd46f4b5b42e63a67ab796", "dsid_fc36d1d60e7e4b4abc7db84629563b7a", "dsid_31b9fc3dd6db4b7e93b8d4596b09ac2c", "dsid_44ba18f330294fac8a7b61e938e4598a" ] }, { "question_id": "qst_0138::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_db931dec10bf4b10829175533605aa0f", "dsid_2a640cd6a0f44f07bd772d630f650e34", "dsid_221c49190a14450595ca5dd9ab1d801b", "dsid_52a32f2d0cda4647b7fa12dda76ed277", "dsid_d81ca6c0b6104ad78c27a3b2cebecce5", "dsid_31d91013efc346b889a211f3429fdcef", "dsid_2435609e0595444ca054fe6069bcc905", "dsid_862bb899faac43feab4f92a12d320e7c", "dsid_033acb55079b4d42aac8ecc7d6b3b511", "dsid_0517272f49c544fbb3eb01799da8dd82" ] }, { "question_id": "qst_0420::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a3303a13cb834f8bb6092e3e3febd30e", "dsid_75ae758fddb8494eb81f572b3f107ddf", "dsid_73e9ecc15e4c4f41b73acd36d3244519", "dsid_858fd59a63d64f5e81a2b8482ba64dbf", "dsid_ddb75b4679ea48c4904745f1feac5989", "dsid_91b27b1e5859467ea4d043fea106b850", "dsid_075160d3095a44f1ba40bcfa408f839e", "dsid_53cffcf349004c2687f0bfe1d62bdaae", "dsid_6f8b4127cbf342f8893234fbd7b44b80", "dsid_f211c2195b4e475c865f78c4948a2c78" ] }, { "question_id": "qst_0462::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_df79433aa56c489d93efd96e27dc1fc8", "dsid_e62a534321a24da2a93a003a9546804b", "dsid_b72133b826ef4dbf833b558aa9e7b4f6", "dsid_da7e13b65f8a42b4975b44b2db0a36e4", "dsid_b52bebd0520d40b0a16a32940d12c9b2", "dsid_a3eefbb375474efc91283e84915b132b", "dsid_6c4672adfeb646faa537a68b791e9787", "dsid_adc5bfdf4db74888b02a97baac83d2f6", "dsid_56ed3f7cc8c24b14a9f68078529290b9", "dsid_c1a88137436042189c6fb011ece2b760" ] }, { "question_id": "qst_0236::semantic", "question_type": "semantic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a9bb70a65894469ca2cc97a8f9dfd7e6", "dsid_9774c65c77884ba68537610017cedda6", "dsid_9aac66ad84464f4b8e05ba6df40ec66a", "dsid_189583d1a5ed4513a4d709a037098a24", "dsid_0183508e707b4cc1b5a81ac8634339b8", "dsid_ef0efd89151944dc91bbd43d90d094ce", "dsid_fc77dac56ce4424c98cb7c934d56fa70", "dsid_65b19a254af64a148c357c330b157ea5", "dsid_ecc2912735b84233be0196aded0d63b3", "dsid_4cc85ebcf4a2418a818749c4d6a33402" ] }, { "question_id": "qst_0464::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_199789e68ef741bca44a24d535ed6c00", "dsid_1124747125dc469c947cfaec10a4b6bf", "dsid_1bfd5cf3ae504f46b081ff3082c99b6c", "dsid_d93d7f868f0b4bce922537efeff9cd3d", "dsid_ee3d896039da40b0b0f357542dca5447", "dsid_3867dfe05c8e4f7aa1dea7af561907eb", "dsid_94b6ee91affb440180c9e7f303173668", "dsid_872d712155dd4c07a36fb8cec22d43da", "dsid_bd84fa64801f4bab94d29300400a034b", "dsid_0337089cc5364efb9ac772924f86b584" ] }, { "question_id": "qst_0373::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_16996cd96c1c4fa1ab399cbc446b186f", "dsid_5324918997644ea090703e088b166214", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_e5fe3fd0cf444668a64970ec00818f46", "dsid_b837e1140d1a4c3d8efd2f42d9b6a799", "dsid_1a3a408d31b649cf8597fd1fefbc24fc", "dsid_c83811391709468fa9534e6495c205e3", "dsid_5555c6b607ef4785bf8df9d0970f743a", "dsid_fea296fcd6ec4064a00ef13353a0c72f", "dsid_fcf393d14cb149e3b195e893e9b263d7" ] }, { "question_id": "qst_0382::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7d347905549f4456b0d2226b47e4aa68", "dsid_d04dea2d736445c3af6934b900a12ef9", "dsid_c1a88137436042189c6fb011ece2b760", "dsid_f7cd9aa435bf4bc5b4c32167fa388e33", "dsid_84742ad3d14d49b2b14e9ee51592ba33", "dsid_330be20dd4224d8dad24eb633121ba06", "dsid_43dbaa6340ba4f6b8f5bea0fe2354bc7", "dsid_a7baddf163ef4946a97b4765bf1cf08b", "dsid_602422b841b94f5c8089555015df0ed8", "dsid_91abfa2001d747b3b570e8abce00ef4d" ] }, { "question_id": "qst_0040::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_43ecb807903f46848e8d099b6c8beb04", "dsid_a72fc5197c274cf283b3e18e40c4a275", "dsid_615b95b58757481a96975cbe5ea4f65c", "dsid_da0e55a5772945deb0f611f5d1d25c16", "dsid_5f0ab47fe8de47a89bb6668f8722becf", "dsid_edff730f9617429695b244d94952481c", "dsid_38598d6cc5224f93a6bc9c823c4f7579", "dsid_84ec014ad9f14f1481b45484da41181a", "dsid_992ed188a41948c9a4c3c93005ac351e", "dsid_0ec5dc3d1bd7499985f080a1102cfe87" ] }, { "question_id": "qst_0323::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a2c7dc70aacc470894c953b6c7499228", "dsid_41f825d6398e4a6c930f2d1f6de7b86b", "dsid_b6a1828946254f6b99c84cff740a9d11", "dsid_6cf7a852191c4efcb1d19c5925b3f2b8", "dsid_e98f49df839f4283be1a9e336f7eb5d1", "dsid_f70e8011de994a9d956fca708445fb5e", "dsid_3fb8a400e0cf42d0b19dc134e2b7f46c", "dsid_e5040a0aa02c4af8a3b4681335b5e328", "dsid_3ce244ef408e428bb409c59b283f3aca", "dsid_4036a41749864632bcda458ed967cf0f" ] }, { "question_id": "qst_0003::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c5288dd4874345adb80f4a71c9a18773", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_2b2d761eeb32477599aab4faf48c6c1b", "dsid_329e7ecd6b304c30b61a30e07919e008", "dsid_562213b6a37e4912add20e642b9f9a15", "dsid_5bb7fcfd6cb74a60aa3437781a30b1ba", "dsid_3f7f32874e9b4ba49ff4fe28c0ef5de0", "dsid_80e095c2dc1745af86b80d24e27a9857", "dsid_59a4fe197fb541f6b5c1dcadc68e302d", "dsid_6c1cf94ad59742a499293dea5dd7fb28" ] }, { "question_id": "qst_0150::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a75a4d9aa7d940009b7966188753d520", "dsid_f940e7c95688406cb497b0b1dab44e99", "dsid_da28108badc24c84bd2eb13aad8bd9d8", "dsid_4b66d6d13e614ff89617822c0160e94b", "dsid_5421ce60ba6148ddb85d6702458f3499", "dsid_e7702842e5334127855135d300ced573", "dsid_e36237ea432449c9adee86ffba3bfcfe", "dsid_782c61bf8a7040388dd565a27e7fd837", "dsid_e7666ea551014373918489571c79ce5c", "dsid_5fd9610c99c34ead80af97ed8dbdde04" ] }, { "question_id": "qst_0134::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_4f0b59f1c83640459247deabe08dbf23", "dsid_9e0b361983074b699ec90b92af18b182", "dsid_b4f5d647ef7c44389b4052782c959bbf", "dsid_5630a13942984b44ab4a6d8c9e286e00", "dsid_2b7a534e76c7405db71d7e770e31eb66", "dsid_05837e0efe7c452dac714bca05ec9335", "dsid_d885f7fe13ec469d8b6ac95d64c2fcec", "dsid_9bf6777d45b0433b947efa6bee592d81", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_85ad38b993154e5b9360a69553da204f" ] }, { "question_id": "qst_0453::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a8a5eac6e7a243319426769963506277", "dsid_99ee2f1f290c4e6881ca562c61837d2e", "dsid_bee0aedaea6f42d6aae86854a6057d38", "dsid_6279854275e44cce9115af8286fdf38b", "dsid_9d16fd62a21542f49bf3e88b2bc9793b", "dsid_1878e91704a24813a12f69b40444f168", "dsid_b27dd01734604ee89b6a911d335b76b4", "dsid_adaebcf9535f47b5903bbaa5e48fd0e6", "dsid_7cfeef2fa6dd4b768bad2303c4467f11", "dsid_992ef49600a7462f8a92def10141c357" ] }, { "question_id": "qst_0423::conflicting_info", "question_type": "conflicting_info", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_d94f119e2e5345b7852f4c0f63927155", "dsid_8733ed2132dc4232b2797c1d9a167cb9", "dsid_dd0fe40a2c71489793d566e46e95bb93", "dsid_03042cec7a2344769bca5a624091092e", "dsid_385fd2388669469e87964b25d6428871", "dsid_1dceec2e5ce44dd989b690dc690ef9f2", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_50f47a9662ea40858feb7bd975071c96", "dsid_ef439cbc87a3404abdead47aa815515a" ] }, { "question_id": "qst_0038::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d04ffbe63ea44224892533eb0fe95bc6", "dsid_06e5984cb6004090a898923bb50635ba", "dsid_a4abf484b679498096d46b21d91b523f", "dsid_2d7d6701e297486e80d23fee93603f03", "dsid_8d8ffa84a0334ff080ff3a2931a431d3", "dsid_59217895119e4a2db2055cbbcbade358", "dsid_a418a258f0d34344ac95005e64ae6b28", "dsid_572a82a8b16949be8c5445f60fadd851", "dsid_f48981bae6bf4be7a9eb5b3c1f3b4ddb", "dsid_672bd385942a466798c7764d484e10d8" ] }, { "question_id": "qst_0392::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7fd136dd100c4c9cbbe1958c94c26152", "dsid_dc5f484a1a844bc2aa450f949a449ed8", "dsid_258679d566634d4092bd5393f349ad04", "dsid_feec98971ec24266b32215046c6a6e09", "dsid_1572255f25394d7ebb868baae4f7ad68", "dsid_a734f47b45294b98bf495b69d251ad8e", "dsid_58220c63dc384549a60a5eb8e400d55a", "dsid_21486610d7314d84980518cb355b6a97", "dsid_9adc7c1c81f54e0a88d0e0bee59a6588", "dsid_c4355f1b52f04596a8b2bdf8226448f5" ] }, { "question_id": "qst_0091::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f8c8a11bcb224dc0a68dcabb8aeb2a38", "dsid_b7e285fa5700488fb87461b56db5bec7", "dsid_308b0d2adccc4f0f8f27ff8a782adcc3", "dsid_9b44585b47884906bec2e2afeab9fcc5", "dsid_2bc3ffe8868640c18867e9b8fd944553", "dsid_9776bd75ffbb4ca7b10669be1512daa3", "dsid_6e435b4d56964862a9696acbcea06cd6", "dsid_abaf40b71c07418ab6207b3f2deb894a", "dsid_dbd29f9393f149fbb696c9a5614dc875", "dsid_ce646062847e440fa50c9838f04f73c3" ] }, { "question_id": "qst_0389::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1d53445cc74b4d7db325cd585736f87f", "dsid_456e37c950de48b49ec504feff97964e", "dsid_a9d291a78fc749758e972678d5f667e6", "dsid_f02a338c2dd546b3b025b5d5d429ad4e", "dsid_e190b0de0d27466fa91bfc18b0815311", "dsid_faf3246a6ce845e6b5df0ae52f439399", "dsid_f80e0fbb8d5f4080bb1350237e1c0055", "dsid_a7b5357d7862433faed17832bdccce47", "dsid_876b1a31bcc7409ab560b9ccbe5a0d41", "dsid_8e8f8a5113c3438e93575304f34aed91" ] }, { "question_id": "qst_0045::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_80a4859760ce4cf4af51f7c570903ab8", "dsid_662978747ddb47338b62f7d87c391ca2", "dsid_3b8d917df5a647848dcd8e3768fe7413", "dsid_0f17506873fd43a985b09807dc594c22", "dsid_168fb762b10c4b77896d7821e4328094", "dsid_f8b7a26d6f68479496f7a05a2298b787", "dsid_fb597c66281142eaac666d0e86ee4cac", "dsid_87cdd3bedba64b59879d2c9e2771cca8", "dsid_2fe6857504b9471fb5954830c70663de", "dsid_fc009de77554459b89e6b4340e8eca82" ] }, { "question_id": "qst_0415::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_05841423a55e4c35abd276d9899b7134", "dsid_2f9c99fc8ec74495b4b08c49a6083ba8", "dsid_f2eac2f1d7864a84bbf1959160e76eb5", "dsid_834de01417d04878b4257f0f03ccbb88", "dsid_b52a8760cae1451a9504edcf9ff58e31", "dsid_28fb7357c34f4620ac6c8d937f0dcfa7", "dsid_1d487c52d79a44c2b40eb19a0db51122", "dsid_2eec0459f3cc4844bbe1556a656650a4", "dsid_6c58ddb8ecf543d89fca8352351dea29", "dsid_b17470b55308417887920a77bc20a1b5" ] }, { "question_id": "qst_0045::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a673ed031ece43c5bc4a0d27a284c8dc", "dsid_3ed876844ccd4cce8f42d3e31c96b015", "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_dd0fe40a2c71489793d566e46e95bb93", "dsid_8733ed2132dc4232b2797c1d9a167cb9", "dsid_50f47a9662ea40858feb7bd975071c96", "dsid_1dceec2e5ce44dd989b690dc690ef9f2", "dsid_509a32fea66b48e991419a0353968347", "dsid_94616da0b07e4b47a5e26279e5f92e5f", "dsid_b87d81a8f52646f98e614d2a6394d7a3" ] }, { "question_id": "qst_0208::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a2e4352770864f679741fd902a51ee87", "dsid_4f10cd8e52964735ab66e40edabc2ad9", "dsid_5ef618fee40741bb9fac512e2448e560", "dsid_fff0eaf808b149ae93f73d4d56dc2dac", "dsid_db75e45174e1428b83ff1ee63673bc01", "dsid_85a183c025574d05b5476634a6d67321", "dsid_0274fe3d48ef4ce1aba0e03171070734", "dsid_dbe9970e1a334dcdb9b6db8bfad6c17d", "dsid_fc938c00d7aa4a0e9630594489095db6", "dsid_957dd0939fc64a5cb7241d73dad94f51" ] }, { "question_id": "qst_0278::semantic", "question_type": "semantic", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_43dbaa6340ba4f6b8f5bea0fe2354bc7", "dsid_5034e5e2a278441dbfea7fbfcd557b58", "dsid_84742ad3d14d49b2b14e9ee51592ba33", "dsid_ccaf626687b940f3a7309e09c1ed5c1b", "dsid_7fefb56c1ba24ab99d25ed49e8f76847", "dsid_13a33311928a4555b16c654cb25251bf", "dsid_602422b841b94f5c8089555015df0ed8", "dsid_e5298124a9fd4bee8242a18bdb3829e0", "dsid_6b9ee26ee41b43189695e7a84587bbe5", "dsid_f7cd9aa435bf4bc5b4c32167fa388e33" ] }, { "question_id": "qst_0189::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_20ed9182867145709feac129172377fa", "dsid_168fb762b10c4b77896d7821e4328094", "dsid_5d54f5f89bbc48a5b06b8dead39729b3", "dsid_5b66dcf51a5948238c3338be191bf03f", "dsid_e552d5c3fc3c491fb40efa4af486faee", "dsid_abfffb03292e4ad1af99d967a0a47501", "dsid_e1b3e01bdd1345b586eb5647477cb34d", "dsid_78555442db264e8bb41349f7ce09e747", "dsid_1ff1ebf46f9b42d78f33dd7c283187ab", "dsid_d40b35f2a00a48e9839cd7ba1157e017" ] }, { "question_id": "qst_0078::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_cfc93aaf69bf456cb669f5b99c09b7fa", "dsid_b9e31a2358c7454ab25d7b424ee8284f", "dsid_cc4c5c2310714fd99962514beac82e74", "dsid_edd9e60a5f7e4f3eb0835259beab643b", "dsid_9f1a05a6016945148a6aa988f0934a19", "dsid_05837e0efe7c452dac714bca05ec9335", "dsid_592217972aaa4e52ab45a166d75e9fdb", "dsid_f79bddf05b964d54b16f351718752e5c", "dsid_fea3382504134502b316ae47b6476f1c", "dsid_ca18eee975d34f4da7b1b880ea5ac271" ] }, { "question_id": "qst_0318::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_521553a072244d199d06a34dd6e1aca0", "dsid_d1c65a017466400e9a891bd531b87874", "dsid_96e315a4f8644c8bbafe05c2e0c0b775", "dsid_4d3b46af4f174805b7eba44d8b3fb5d4", "dsid_d5132fb0dc8749339c644db1ad712532", "dsid_f7fc3f18b5514da49305c3462a2a8147", "dsid_7fefb56c1ba24ab99d25ed49e8f76847", "dsid_9e2e640cccba45f485fae2bf07ede6f5", "dsid_4b9b0134beda4b3f99c8bde8e2d1b0ed", "dsid_5958d02f89fb4504851a6e00205a0af7" ] }, { "question_id": "qst_0003::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3fd6af404fae48e6b8ea5a57875ef78f", "dsid_7251af667c5248d0b9d4480198c90e50", "dsid_c430210ea1e746839398bf61cbc2c3ac", "dsid_34120e1690ff469da2c3b043db4827e9", "dsid_aaddf460f19349d788929b9256754451", "dsid_cdbfbe0595d2418f81dbfa85f64c5d13", "dsid_24b8e1d419804780ba214046b9752680", "dsid_0e7dae8f7c12428f8abbae06b80ed893", "dsid_ce45a9a4d29341be9911c298ac8c3b6e", "dsid_e36cca24c80e4b50994878eb1c66fce5" ] }, { "question_id": "qst_0197::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_e964a683919b4fd9ba304c8a976d8129", "dsid_2fede54bdbb24335bf08a65d19d93de7", "dsid_1b6a17dd05f64e97abd48d43d7e49cdc", "dsid_ee25de12b9594985b03a0106b8c9429b", "dsid_9835602b1b544034bc6272bf2a8c2969", "dsid_01eaeaf6045941beaeaf74e6170aceea", "dsid_1555399090284030afed1c0f50cd8097", "dsid_f5a0ffce1a574f28ae8656fd6d04ddb9", "dsid_7bed0739a8f241219e4f3ab0304916bb", "dsid_2ab066c019ae4cdb97a555082fa0ac41" ] }, { "question_id": "qst_0041::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_4e56d7a078054405842c007b3d3728f5", "dsid_5e4acb8d633147e986cccbe37358e0e4", "dsid_04d9f4e5eda14a2c98f3a97b9f40d64d", "dsid_9c65a74d34004a0ca2955052dd4188b7", "dsid_b91398cabd7648449afa9c62d4ade744", "dsid_25b3d349510d453e90b9d34d541ee27c", "dsid_799a1bb8e7234cdf89834a0a17e04ba4", "dsid_c976027b41f24d82a52c2a6411f6b444", "dsid_64636200159747ba9951af415453fc1c", "dsid_2dcd852792f7475fa27dd7a0a56fed00" ] }, { "question_id": "qst_0166::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_10d018b9cd334ababafa960517492a77", "dsid_de174c9811834eff9929fd6d4efb3d30", "dsid_bf0a48f2f2564bc78dbae6b00ce0fcef", "dsid_2bf4ff734c75441085b56676a0347986", "dsid_245300dd8ba74dadbf1c363db1c94b8a", "dsid_590dd1330a4046e28ac0a2f43b7910cc", "dsid_b74ba3d7aaed4a588bd813af431f906a", "dsid_521ceaa63b844d85940cd637e799da97", "dsid_3be64887cf9c42cb96f242fb6def7d8d", "dsid_fe7b8f93a1ec4557ac29658c12a33653" ] }, { "question_id": "qst_0251::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_25b3d349510d453e90b9d34d541ee27c", "dsid_9800d7f35ee94aae94ad8a650980312e", "dsid_e611d79c110248d9853fb40b4be0e216", "dsid_5b7ebd1666de42f68602a7663ab34b63", "dsid_a6a7dc6232154f32857b3c6396d6a294", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_2a94754480884dd6a23c3080e4a98bef", "dsid_c8ee7fabdcb946dc842d40c552180329", "dsid_85a183c025574d05b5476634a6d67321", "dsid_40b70b4142a74746ab5e07860d44f3d8" ] }, { "question_id": "qst_0085::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2271ae6c393d409093b11b5a6d5b96ed", "dsid_7012b923d3564bb19890be9d17704c70", "dsid_89e24d37351c4706ab5b8b313fcff49d", "dsid_658b19d662294bd5a4ff0d9b8b5ccfea", "dsid_f56c5f89e2574cc88be0ff0ab7fa6363", "dsid_b79bd6d8f17349e288fa2c6640d6d2e6", "dsid_00f81dfe95d641e897af85983a14225d", "dsid_833b0d0146a94733bb7d9d1290ce85a8", "dsid_4d9640dbf6e7414aa2cbc5c423df39a0", "dsid_9067aa81580b401eaa7692342fb820d1" ] }, { "question_id": "qst_0448::completeness", "question_type": "completeness", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 0.42857142857142855, "retrieved_doc_ids": [ "dsid_0032ae6f295b4845b21638f56eae59a2", "dsid_615b95b58757481a96975cbe5ea4f65c", "dsid_6933f9241ef140da9d3bf98be52be867", "dsid_67a8c3300752435296938175e9a1daf5", "dsid_84ec014ad9f14f1481b45484da41181a", "dsid_da0e55a5772945deb0f611f5d1d25c16", "dsid_f922e74ae43b47b2b029526b695bc1fc", "dsid_b6c3a0abc4334512864781fa498ae2a4", "dsid_bd58517ef13a42428e5d14407902df5a", "dsid_cf674e9fc45c41808c9837cf72db6a01" ] }, { "question_id": "qst_0029::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_429267fe79c74918a26272f6c0e48f10", "dsid_18d6b7f9d2614de284aa176dabf09bf7", "dsid_b1b99a34a6b44b74acc644d0da23bdac", "dsid_ecbe2f709c5547739018e89dc8082976", "dsid_61f8e051e2be4134a310805473896eac", "dsid_bd7caad8adb2405eaa7ad78c4ffb59d7", "dsid_4b66d6d13e614ff89617822c0160e94b", "dsid_9a088141571b435f995568199ec4bcc9", "dsid_c14d0841d94c4c07affce6d0d61963ad", "dsid_4036a41749864632bcda458ed967cf0f" ] }, { "question_id": "qst_0349::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_21d6b99dc771418287d61444f373852e", "dsid_fe7a496e795b4baca997e655f9316fc5", "dsid_44cf2d75d70d4365914aa46947a94b1d", "dsid_d3edd63b9ec54523ad12bd9be3d7e996", "dsid_60b6b4cff8f440899bb213cbfb4e90ed", "dsid_6f69a170c5ab486588cf0bd92e0d53db", "dsid_ac94dcaf2e1e4674b2351a8b4177db9e", "dsid_c0c93abadf4c438189e23cb45256c6e9", "dsid_f912a6d60ac647b5a463e981a66e8acd", "dsid_d844cc30147b49c69ea5275b4993b571" ] }, { "question_id": "qst_0324::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a92fe998fc534d57a1d54d5a6294c938", "dsid_5aff8d8242e34bf5b2a5009692149e37", "dsid_0aec41e836124454bf686afc6ae232f0", "dsid_f80e0fbb8d5f4080bb1350237e1c0055", "dsid_44ba18f330294fac8a7b61e938e4598a", "dsid_089917448e654911bd186650ed12feb9", "dsid_48fa74f556aa4f0c9996b53170693e8f", "dsid_81208556437644a0a4dedf168baf81b5", "dsid_806db9b506b04a31b42b04fae256a786", "dsid_c70d63e14aa84195aec43820c018f8b9" ] }, { "question_id": "qst_0270::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_33de625fcfc045dd967dbc9bdde3a8b4", "dsid_9553118b633d450da2494825176bd309", "dsid_0fa842d894d34dd2b5facfa95022d26f", "dsid_62fe5dba760940cc87683dbeb3bdebc8", "dsid_d3edd63b9ec54523ad12bd9be3d7e996", "dsid_79000f2c91674c0ca3995b5b1f79827a", "dsid_865449ae95b044768f33773263bfeaad", "dsid_e7cc77b26210448f9cfa20404b0f5c5e", "dsid_d844cc30147b49c69ea5275b4993b571", "dsid_e91e68b31a6f4935b87d6f74dd07e1f6" ] }, { "question_id": "qst_0266::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_415b72b7d40a405e87b9671946e894d8", "dsid_88b39d13bd584ef8b1422d76994808c7", "dsid_3f1ecaac0b1c4dffa5a680a6b2d34ba1", "dsid_1adfb972e14e4de997bf5b0c96977ae5", "dsid_28234f205ac740018605b17b005ab318", "dsid_13c085cb55f440618d16dd294c7f26e2", "dsid_6d5888fe062d46958864cc8e920d4d21", "dsid_3ebfd94588b046cf909ec52228a1dc2f", "dsid_9f513c2a203b4de09f8d9ded40c40970", "dsid_ce9655fadafc4c82bab8103978f6d89c" ] }, { "question_id": "qst_0247::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3c18e45fbc064bdeb056d94d621846f2", "dsid_0c06ac6db23c49dfbc8ad8049c40ff89", "dsid_96d6df73f64f4f5d904777f2a46091c6", "dsid_0dc68466ec674615a70764c82974a669", "dsid_1abd1632541b427884b6e64c77fa0b77", "dsid_f30a7c8acd944954bb58f1243da983d6", "dsid_de78a2c04c4e4ac397f3fdbca2647dbc", "dsid_5541a810bebb433fab50e71a0a1ea1e2", "dsid_5362867104ef4491a829aaf767a02ce1", "dsid_f23926802cf44ae4b85c4410682eab90" ] }, { "question_id": "qst_0239::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_48eb0d8c888243e8bb38518b5ca56cb8", "dsid_de78a2c04c4e4ac397f3fdbca2647dbc", "dsid_06f747d3d62946dd91bc664271a256cf", "dsid_9616404789f14fdc96fc92c791eaed77", "dsid_60903cb34ddb4fae9b5ea1830dae1437", "dsid_d295b7207fec4a6fb656f3e2a565d4e6", "dsid_fbce1c2e297047f4b87d6fda506b89ac", "dsid_299a1d5575b043c9984210631f2ff8b2", "dsid_feeaf3286a944efb90ff7dc8dc6d20f8", "dsid_e33a7390bf664b77997c35fe4296639f" ] }, { "question_id": "qst_0381::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_865449ae95b044768f33773263bfeaad", "dsid_d844cc30147b49c69ea5275b4993b571", "dsid_c0c93abadf4c438189e23cb45256c6e9", "dsid_81323a96121047cf98e958463e33d80a", "dsid_096e4752a7ed47e680ddc1fe7d3488dc", "dsid_d3edd63b9ec54523ad12bd9be3d7e996", "dsid_557c47d8aae543afa40b6109b257f863", "dsid_53046df6de1d411aa88c6ecb288d3dc0", "dsid_aa32fc5e084a4a35983bebb576693b7f", "dsid_a277ed2dc74c4bad82bc71b534b6356f" ] }, { "question_id": "qst_0393::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_90218d9a9af84072bcf3ff7bb90b5fa1", "dsid_4b15b253ca294377a12d12651ada2168", "dsid_87b573ebca8f405aaa06cd8b1a4b7745", "dsid_4fdcb683b3ee40e89103eb7d384eeece", "dsid_bb592df92b894a9eadd6512774156de1", "dsid_e79d24a687f54e59a6729f5bd4d92240", "dsid_5ff47f3157204b959cb7c6f1f6842349", "dsid_c6620a583abd42e6a60919c55c1f2a4d", "dsid_4a32026b13fd44b5a9d65403ab8efa16", "dsid_9155c08f9405446a9cc51396833752fa" ] }, { "question_id": "qst_0023::metadata", "question_type": "metadata", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_04e98bd61eac4ec89e3fc13c38de1b8e", "dsid_0a6c6e67d89c4cb4b053a7c04d780d97", "dsid_8db78e40149847bd95bdbcd74dd02eef", "dsid_4255f3f005a44e8e84d7c8e10e6be012", "dsid_f837d82387d7481782bf30ac7e26231d", "dsid_190d9732b59b411d8a6bcc59fd61b309", "dsid_e6fb371688bf4c5ea1fdef9722253dd2", "dsid_9fe2046e56e04a889a75d48692017405", "dsid_a2d5f7ee89c14d4ab02e82aa94f22c18", "dsid_9ed01291d81947f49500df5fb28ab724" ] }, { "question_id": "qst_0380::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_be8d2802cf3245088a7e7f861f317b31", "dsid_3e7cf5be42d84f2880858c726b65998f", "dsid_089917448e654911bd186650ed12feb9", "dsid_36cc238e32514fc09cfad6e7d1ce7364", "dsid_2bb0a4d1958848b0939ed64cc2b911ff", "dsid_d2e228e29d064a32afac866e030fb885", "dsid_8de05ce07ac64e668012839695c902f9", "dsid_a860233b7eab440bbcae4b920299283f", "dsid_a67fe17a8dfb4b71b074352afe3fd471", "dsid_a526f1e13b104c549e33244d0000c87a" ] }, { "question_id": "qst_0124::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_30a16502953a4088b59266c8990d52d6", "dsid_715031e81f3f436fa0282fd0f248a716", "dsid_d85fcb5f989b44f8a51723a07be79e45", "dsid_ce2601ae1383462581f4a2c7f4e96ac9", "dsid_f56c5f89e2574cc88be0ff0ab7fa6363", "dsid_92798ef75b3f42529818d1f0acd636a2", "dsid_8aecfa9775e74f61b9f53d3dc4362c50", "dsid_2c018483eefe40f7bf1bb4008daf6a83", "dsid_451062cb78a04e08adb918f8c77a3748", "dsid_1fb4e5df53d249458ca21f421b222d7d" ] }, { "question_id": "qst_0144::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f792e2a148d84e9cac998b2cb5d452f6", "dsid_061c09d8eef649eba85cd1421e28a41a", "dsid_dc2ba8d4ae3848cba5e52cc3f393156d", "dsid_8f07697382a54436afc36dd1b541f653", "dsid_929598314f834ad08cf48642b88bde16", "dsid_ee6c23c178dc428bbd71af6526c70251", "dsid_04f1e0942b464d479d1fc081dc69c84a", "dsid_418533ed4be949989ca92172ea2f9ad2", "dsid_616e3011a23b431da1dd3445e3002675", "dsid_acc33ff059404020b0ff26e19ad2b107" ] }, { "question_id": "qst_0444::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.6666666666666666, "retrieved_doc_ids": [ "dsid_117e5a150aa94adc86295786bd6b7843", "dsid_5cde26f232454906879617dab1800fae", "dsid_40471ba4cd3b430eab3ac0877eef0972", "dsid_2ad75f27c36b492f847d5b9c63492e4b", "dsid_918e5837f87b4368af831ce5d6bd71f2", "dsid_5a234f5c66d14a4594f966e394f11b7c", "dsid_87cdd3bedba64b59879d2c9e2771cca8", "dsid_b414948508a24e8ca0df504dd7944601", "dsid_036427f863ee44a0919827cce14cbca8", "dsid_9c6c596e73474457aa45c13bf8ee33e2" ] }, { "question_id": "qst_0359::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_617d538c79334f4b9ddfd5c50c05ee4b", "dsid_d67b4ea2d8834799ae617b1ccddf1f6e", "dsid_5b0c65ca162a43a0938bebabe9b1d53f", "dsid_2d0330be987e4163bfd9c42c69bc1e40", "dsid_ae455bffe226412c9973991ef2d34a72", "dsid_428905f63d6749b0a8b4809a65cdd6a7", "dsid_04b24bf957c047f4b7d8ceefdfde8290", "dsid_a0ca7f3e5d6b439a8958d2a79bb5b6a9", "dsid_0bebeaedbe794ec38ae1d6f4ddd5c2b6", "dsid_54fe07e80e514a9bbeee9f1e88274394" ] }, { "question_id": "qst_0131::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e9dd560753924c1e8438cdf9c2afcf68", "dsid_2a94754480884dd6a23c3080e4a98bef", "dsid_75fff671684246fc80766d44866bf248", "dsid_4c1c5fb53ca2432995ea42adc4330fca", "dsid_9cc9a4cd19204e09a35ebfe787dd6f48", "dsid_1b2d96d6a5d44eb88aff2562a47d5537", "dsid_b5d228651829498380533cf69504efb1", "dsid_25ad54cb98814d4397d07a512ac2d16e", "dsid_60f1f0a85cd24c1bb471319b01cf1022", "dsid_805dedb386fb4540b2f691618925a0ef" ] }, { "question_id": "qst_0449::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.3333333333333333, "retrieved_doc_ids": [ "dsid_5cf1a6657f1848968b2fd154066950ae", "dsid_ed6d086c9faf4272b0ee20d3fa4b6180", "dsid_8dff963a50774cdd84131d9ae5e4f4f3", "dsid_fe6c423c71f04547b10f617e4a01af33", "dsid_985aa0711dc143ccb160d4dc2f0245c4", "dsid_2d3e6d858d344d1c926c1440283aafa6", "dsid_666b6831d6b946829745d9068675d220", "dsid_5fbe3581c500430e8602422319bfed9a", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_64eae267d09b40069dc92a28dca82dde" ] }, { "question_id": "qst_0162::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a3fdfc45905f46f3988ad13f55ecce93", "dsid_3042eea9b36942ae8084eab2f28c3012", "dsid_41d08aec62844f0683786ff59933b1f4", "dsid_e5c22c12b81947e99bb651ec1ea25446", "dsid_dad29972be094a0292b07f19d837acbc", "dsid_5402831508734a51adf649603be4b075", "dsid_c1230319bf514f689a4a17c65eed6764", "dsid_b019c331312f4dae86ce8b5e2029f023", "dsid_3f4c9ed691ef45d4802bd9c9abd762e3", "dsid_ddd20058532a49bb9264d6b0fbf021aa" ] }, { "question_id": "qst_0034::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1867cdf06a00437194b3250c75321a92", "dsid_28e40386d0294fc480b93ad486e762e6", "dsid_2a7fde99135848b2addbcdb45d8845b9", "dsid_9bb02263f1084f69a5749d89f7c1dcfd", "dsid_c27483a128094c9c8cef4eef00074763", "dsid_deb522245fa341638a0125628c263a0d", "dsid_4255f3f005a44e8e84d7c8e10e6be012", "dsid_f3e6e070f705458c9016f2ce75ae035a", "dsid_0ae9b752fef446ec86e376e2dea49c28", "dsid_cc0f1f6ef452440d8d22000e962682f7" ] }, { "question_id": "qst_0238::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e528bed025b44f71bc37a54f5a9aefcd", "dsid_e54ec48a2f3f4ff68589a34a7b387f79", "dsid_7a2cfe4b7e574e6995f33e64bbdbf426", "dsid_38711ba1d616421897b8581b5b11dec9", "dsid_89c0c31e220c4777b15c3c6ff0eff61d", "dsid_bf8681ce9427455299245cb2ee67b4a0", "dsid_6940b302b9994093b54bcda2c00139bd", "dsid_7ab12a0c235947b4a69264e27b2dcf07", "dsid_1a90f55138ed43f4a928673dd0b36bee", "dsid_a2bf755670ad44e09b42c11e9d73e5d3" ] }, { "question_id": "qst_0057::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d740645aa8ce42059a6967bdad866b82", "dsid_718a5c08e1ef47d38634c3623ace0803", "dsid_10a7359bb6ad4b959ad2171abdb9f620", "dsid_6f10b578f17e4d8aa622302ff880659f", "dsid_089826a919084ec38ebe58576c17add1", "dsid_28fb7357c34f4620ac6c8d937f0dcfa7", "dsid_777ee2eac47a409c8437a703e7be654a", "dsid_b35f08478f9a48f3b04f8a36ac3b158d", "dsid_b52a8760cae1451a9504edcf9ff58e31", "dsid_18120dfa90aa43daadfeb530453ac50c" ] }, { "question_id": "qst_0268::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a990e708840f481795741cfd3fb55691", "dsid_32d162435ad34eeca3861eeca4463879", "dsid_d885f7fe13ec469d8b6ac95d64c2fcec", "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_a7baddf163ef4946a97b4765bf1cf08b", "dsid_5d8045f7d37d41b99605880834c17329", "dsid_d9fd42b7941949e4aa597871651aa970", "dsid_576496e2926e4747980636f3ee519243", "dsid_0488c203b7094b0d9f70c13891c7bef9", "dsid_15a9d93cc59c4241a97a7e72f749a216" ] }, { "question_id": "qst_0261::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c9a1ca32dae14498a89c2ae1b1d5bfa8", "dsid_03bbbaa2fd64496395b8f6991372bcc4", "dsid_43f8171daeba4affa59a42e4616e1891", "dsid_6fab1605dad9481a80e693ddc5fed63e", "dsid_6479575dab204bea8e6dc3c8ee9863ae", "dsid_fcd790a504a84e7b83f889997e2b9b3a", "dsid_dc7580d2de0a4de1966ed326544d5dfe", "dsid_deb522245fa341638a0125628c263a0d", "dsid_a9b9485b90cb49c78fc96cd888e2e5af", "dsid_2a94754480884dd6a23c3080e4a98bef" ] }, { "question_id": "qst_0021::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_929598314f834ad08cf48642b88bde16", "dsid_dc96257dfb474c3a9c0d086765c039ea", "dsid_acc33ff059404020b0ff26e19ad2b107", "dsid_b607b02d987f4e53853d3afead88de64", "dsid_21a33756e6d94f65a7ae1f1d096cfb82", "dsid_ee6c23c178dc428bbd71af6526c70251", "dsid_d39546f69fa44000bb4ce512825629e1", "dsid_dc2ba8d4ae3848cba5e52cc3f393156d", "dsid_f6fb5cdfb2a64495a66fc1d001e70bae", "dsid_2d0832fecc194b69b8dbc3b4714b7804" ] }, { "question_id": "qst_0298::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_408c94bb372743f4a622da1d30717c83", "dsid_e787c48ea74e43968a6dbb4674d4db86", "dsid_719f6715eec6487ab8a92f88f8d83257", "dsid_d8591d8705054963aac830b0386ff68c", "dsid_33c31a746b7b459faff2c646ec5380a1", "dsid_033acb55079b4d42aac8ecc7d6b3b511", "dsid_95612402ea2e4481aedc7c46cdbad170", "dsid_25b3d349510d453e90b9d34d541ee27c", "dsid_214e181294374468a48f127e93169868", "dsid_85d1214a19b44f779a5c844647712756" ] }, { "question_id": "qst_0313::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e9f22ba025e34bb6b7b8c61d26a38a19", "dsid_7f2a6f2d2de34f94a5b6f3d5d0d1f6a2", "dsid_b4f5ca2611ec473c8affea7e0fcfe4bd", "dsid_87fb9790b17347268d4e0672af499c40", "dsid_72f9a72f30184cb8b1093389c0c6a8d4", "dsid_1988feff8a414541a33ba3ac70400ac4", "dsid_49215834c17d4f0c98d4ad11f62f0c9a", "dsid_1a4dc2c6f4484058bebc7c22da836131", "dsid_37befab212ae4ed9be7ebe612d942658", "dsid_6849218bed8c4f169cb6dacfc33a7c4f" ] }, { "question_id": "qst_0244::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_37c24c697d9d462fad0e3afeeea9d8c0", "dsid_98d6747d9d864df2b1e7ff5607140759", "dsid_f6dec11049ef429b8d0d8e2b903012d2", "dsid_d96ab96bee80492cb60c8b8ec8884d96", "dsid_dced9d2508b54c45a65b20af24dc999f", "dsid_19d5d2f6a4cc40cfaa6086ca39b2b741", "dsid_bd7caad8adb2405eaa7ad78c4ffb59d7", "dsid_7b664a35119744e9be84ef1f403e7494", "dsid_64cab713f34b47c684a50f2315e62398", "dsid_f6309f8a2a8a4ca99d9efa6bffad3567" ] }, { "question_id": "qst_0077::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1dffa4bc09594854ba264a78145d4dd1", "dsid_88b39d13bd584ef8b1422d76994808c7", "dsid_7f9e5e1c40184e0b8ae1040e9ea3692d", "dsid_cc6747e10e8c40699f5adcffaf3a667e", "dsid_aae640f26eeb44be89a6c1363c8ce3bc", "dsid_3ebfd94588b046cf909ec52228a1dc2f", "dsid_730d222c7ac04c56aa1a21081a7479ca", "dsid_9a457f17747d464b988c1726d377b982", "dsid_1a856cfde7344e08a209e61e44b8f884", "dsid_3ad691e18be849d897506f5276c24a75" ] }, { "question_id": "qst_0424::conflicting_info", "question_type": "conflicting_info", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1a90f55138ed43f4a928673dd0b36bee", "dsid_fe677bbfee3945f28b37c238624473ed", "dsid_658b19d662294bd5a4ff0d9b8b5ccfea", "dsid_21a33756e6d94f65a7ae1f1d096cfb82", "dsid_616e3011a23b431da1dd3445e3002675", "dsid_7f2a6f2d2de34f94a5b6f3d5d0d1f6a2", "dsid_8dd74955eb4440c0b573eb53aed54d6a", "dsid_20230cd4f1f24d8e9db376188b3bd21e", "dsid_f039509796f842cd84ba3bfb895c7132", "dsid_08a5ff6ebb3048b49b54576b304909aa" ] }, { "question_id": "qst_0401::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f7dc91cc70e14c94a6215a13cd829cbf", "dsid_c41f0dce7e8e4323a8cc8199e76056e4", "dsid_c866c6dc5c6d45a782337788c3424167", "dsid_5b66dcf51a5948238c3338be191bf03f", "dsid_f940e7c95688406cb497b0b1dab44e99", "dsid_f1b208f3637e4987aeab371242a5bea5", "dsid_e77ca0e1d4774d73a7514a161736b8ee", "dsid_505906cfd18c40b6bad41ec9aa78cf62", "dsid_18d6b7f9d2614de284aa176dabf09bf7", "dsid_fc0e5f714fe543739f20c253d549460a" ] }, { "question_id": "qst_0053::metadata", "question_type": "metadata", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0dc68466ec674615a70764c82974a669", "dsid_c3be99dbca144db488c037dd2f0831ef", "dsid_62fe5dba760940cc87683dbeb3bdebc8", "dsid_99521ce813c340549ab5d4b627c2b806", "dsid_08804d96eb254c46bd0e9b6a1dd30e6d", "dsid_b6d3a7230bf8449591d25efac3d62ad2", "dsid_1d68e5ab4df8461b9107b828530ed201", "dsid_94de9884c49b4d8a9d8578be1186d38e", "dsid_8078605188be4ac79d994ef28da71865", "dsid_80b040ef3b4a47b082062e490dd845f0" ] } ] }, { "corpus_scale_size": 50000, "condition": "raw", "hit_at_1": 0.425, "hit_at_5": 0.605, "hit_at_10": 0.645, "mrr": 0.504, "document_recall": 0.6148, "per_question": [ { "question_id": "qst_0102::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e044274a9d184700bb8cdf9ddb6d9d02", "dsid_8e27ea15627346d4ae210a5ac28bd2a2", "dsid_0b98dd77a056407aac139a12a16302de", "dsid_2bb75fde3be24cfcbb35ecb1c14dab21", "dsid_639dd0469ca34277a35642bf3a09d3e9", "dsid_af125e875c004628a365e906c6d889ea", "dsid_bd4cfd315c2143a0a1199056b9bc35be", "dsid_2ab897bf758e4c40b42b75fd0cc020fe", "dsid_3f7a33cf82d74ff58625c4d4288f2aee", "dsid_9384397e3c294b0a80d0e74d12178c1f" ] }, { "question_id": "qst_0002::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ae3a5380355d4db983aad19c6939b26f", "dsid_c1b16f241ebf46caafdd845edd2c3da9", "dsid_8eaa2785974140b3894297324977a8ff", "dsid_90dca687911e4baeb9ad961be8f0da4f", "dsid_4b2ac6984bdf4737a18520e158b04453", "dsid_260f077d25b04fa1be926188e8cb20b8", "dsid_6ddbb46c266647f4922b8107fd6ea395", "dsid_857e8bf5ab2b423e9bc8bd5f6ea1583c", "dsid_8935178ab5b9485a9b7fc0f048bf0067", "dsid_592227c860554d9b8865d8079d0a7b26" ] }, { "question_id": "qst_0252::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_15e23a53f5124b67ac74eec12a9c59a0", "dsid_6f7992198948439fba3b7194cb3f8b4a", "dsid_5e5365d296024949a26033b26fa169d1", "dsid_7a1c8832d9aa4a2cb830f4824ca8acc7", "dsid_6e2ea493db354c56bc26050e00b2dd2a", "dsid_7b4d5aae7f064e25aa4e0138a6c9d4d0", "dsid_b83c94333b424640a913e66e59433410", "dsid_a5b2109e90b14ba6ac841ff8dec11073", "dsid_d8e75effbe37456183c936ee38d3a302", "dsid_acb118e52b894a05a85254574d46b0b3" ] }, { "question_id": "qst_0020::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1c53affffb4f477ca476bec3138b60eb", "dsid_87fd5087a63444fda38fcdc8dbd90ee0", "dsid_22b0fa396e0c447ab845a50dd70f59ed", "dsid_292bf5b28c4041ff80ddad285249c4b7", "dsid_3abfb1b3210d47009ff9dd0f1aa6ec86", "dsid_c4e18f4f1d0241508865776be8a0fa01", "dsid_4b4ff93a17874621b8e10de846cc17ce", "dsid_2896936b49d645cf9b567cc68247d6e4", "dsid_23f92404aea54fa591224bee0eddd145", "dsid_8ee929aed2b64a178f26b7521fa7b495" ] }, { "question_id": "qst_0113::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c53d4b486a1e485186a7e4b9d1c3f3ef", "dsid_db9f6f36e1e14736a36e4282ae5ca113", "dsid_3812d098cb404e3ba3d4ecf51436271c", "dsid_574c6f4f460d4da893026879eb4eb3b3", "dsid_b11b070973674f8e985739d9cae1773a", "dsid_258577e1a47642f5a1a85902cf2290eb", "dsid_0c5af5cc84874d8f94114499d6790f75", "dsid_2b16c1b556b244488fcde4703f85fe2e", "dsid_5213609e26c644c2a04edd093bb0a65a", "dsid_8a64a0f74eac488785ae8e9cb1201e11" ] }, { "question_id": "qst_0204::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d0660866e36b4877b1f828b395181c9c", "dsid_9f464c859711479bb215f437939e6319", "dsid_3f710f583a1f40ba95ef2c577641f88a", "dsid_4565a39692a741e2bbecc7fcb9cef963", "dsid_f0248044321c42049e4e6b56883b8223", "dsid_3237666f5dd74914b1bb5cc5dd7630b2", "dsid_c6cac1cc18634fb3b27e17ae239fc1c9", "dsid_acd812cda19c40fa8c5a9c044aacd6d6", "dsid_a94ec0f2efac4f4b8a5efb9086c30259", "dsid_52a32f2d0cda4647b7fa12dda76ed277" ] }, { "question_id": "qst_0089::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ff16172c98f44e4f8c4b29dd3af2b734", "dsid_368c2e67e1874131a10795d8e5de5827", "dsid_57071f1894c840998c562ffeb1d2e8e8", "dsid_1400062839f747889e0b8a2a5f7080ae", "dsid_37ca232bc6664359b9468d373c8f6ed2", "dsid_c9770748e4f94e2aa63b91d342ae5241", "dsid_af565591ed904f23a2de393951faea92", "dsid_4dc4ffb08e894b698209725ba7762eb6", "dsid_cd0a915e779b494d9a1327269746d51f", "dsid_d5deeeaed3ec4432a137c10a0e731989" ] }, { "question_id": "qst_0006::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5b3278541a824fc7a21d24dc2d7bd5ba", "dsid_184be937d34a412ab5e61366d54d8ed6", "dsid_adf4b0636870464f9fd379dcab147388", "dsid_4f2844ba27d9489d882e1a332db2c618", "dsid_f60a7530d68b40c3bcadfa26f42e48f6", "dsid_e1bedcdb2fd34c9a9837a3dae2838a5d", "dsid_782f16eb2e50458bb62c35dc21c0702b", "dsid_12b40ee174cd4ba2abb680c627e94bbe", "dsid_e3e317dbf4ef42239197576caad0f273", "dsid_1dcd3f9557c042dbb6e343b454199a45" ] }, { "question_id": "qst_0106::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d6ca2f261ef548b792cb7ac9a2d3f7d5", "dsid_c56494f96ed34157979360e408bf5238", "dsid_6f69a170c5ab486588cf0bd92e0d53db", "dsid_13dd8019c75c4eac9d644cedc987f63e", "dsid_3865bcbca3404c6f8f2687268d3ae93d", "dsid_760d1b38de7e433da4223b43ffd25642", "dsid_c790406c34ce484b8d00a19dbeb476a6", "dsid_c921f79e35a64d4495031651a65d8d4f", "dsid_e262877965414d6f8db776997d388578", "dsid_4d16016742fc4213bbc90d5e2dad5412" ] }, { "question_id": "qst_0269::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fbbdefae0e6b4f80bd50166c0f2395a7", "dsid_fad07fd45c964078ba757dfd8d79ee51", "dsid_aa32fc5e084a4a35983bebb576693b7f", "dsid_f466dfe813bf466580aec7b0b691fc42", "dsid_25cd189bbfb947a0af8343b5ff382d0f", "dsid_a9566b41322942fa8fef145f63c2a7ae", "dsid_e86ea669ce674d9e87dd8712dd738349", "dsid_1ce4addffe764b1f85168957968063aa", "dsid_ad243065c4c24e2a85789e4403a2cd36", "dsid_cdaa439615ec4fa5a3b2f4afe904e474" ] }, { "question_id": "qst_0025::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_012a01961ac74fd28a829c2bb7850573", "dsid_8401a32f4d644ecab18ee24ad6149873", "dsid_01c7aa20e6f447a89af72eee77aea203", "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_f78be6e8423e47a497acece99aee2cac", "dsid_73575cb54e0d4ce384ea9b5006905026", "dsid_cb65aed62384450f84f76c31592f233e", "dsid_cd7a87ec3e5d4cc0a657c64265e94e19", "dsid_4700682214424a7aafe594325c48459b", "dsid_c22869a06d0a4cb89e595c2caea91299" ] }, { "question_id": "qst_0054::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_bea915b9aaab42e88c5b1cd53370d90c", "dsid_4210fbb4f63c404c837bc9c714c55a99", "dsid_c64c29822dbf4194947535359c0ba77c", "dsid_17bb4c84941e4fde82a79a0b1099c0da", "dsid_b9f1385353614254aca460da1bfcccf2", "dsid_b8558b8d57494d1ca36ca54d75b4f538", "dsid_6aedd6f3280940acb11f7e21cd658e7c", "dsid_0bd69d18b67d4cc1826626d94f0a69a4", "dsid_d4de13b87a9b405e915c48e546d2d7e0", "dsid_0e2ba87e6a054a6d84b6c1488fd01ff5" ] }, { "question_id": "qst_0048::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f3af4a0db68c4b15983439460cc73006", "dsid_2a4d14cb1f7142a9a291d2801a53259d", "dsid_a83872e693df4af9be4fd4c477273128", "dsid_e8c3b69eee25487bb1d642d16cb507dd", "dsid_e63cd35ba2764449a94876bdfd4b1ff3", "dsid_caac4d229c984546956865b6a2ed7ad1", "dsid_f78b57c98d0b472198fe9cb25c04822e", "dsid_2bf07e5e51bc4ce4ac807cab8e90b2a5", "dsid_c2e0a6d74d59461286ec29ce7456a1dc", "dsid_e8308a2e5e014b1a99f763abc14be91b" ] }, { "question_id": "qst_0181::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_7df11b381971496da60de2fa837ad090", "dsid_993bd7e86ff647dbb871e5e1442a7790", "dsid_ead5b36435fb4170b49ec5e9dcb07787", "dsid_2f528e76ac7243929106d218ce3822d4", "dsid_ba16d5ba48764a7cbd00c69b344d27f6", "dsid_0fd905b55f43470f99da797c9ecdb2f5", "dsid_f3cf569c1eb448f39cf5c531fffbe405", "dsid_a734cb0f87894b66825518fc956445fd", "dsid_b38d33230bf54c739560a458383f0fdf", "dsid_cdbc1445e0f0477892990d95d032c808" ] }, { "question_id": "qst_0044::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_8107397d5e8e4a028e7f1498880f44f6", "dsid_77747e21fac94ef093179dac9c6a86bf", "dsid_13de211883dc4e9cb2d1fff3b7828084", "dsid_d2995bad08a94af4ac7fba774b5f6b31", "dsid_751e9cf7640b4a7fa73a68ab3c62a9c5", "dsid_dd5f0e9d5435414791786f0ab4cd0363", "dsid_0b17061247644846838334f98b738f9b", "dsid_96dc9caea8af40e5975c66505ed68bbe", "dsid_ee961b1488c548928c5f22971411d307", "dsid_6989ac97d8804c4cbb1d95ff2d1de82b" ] }, { "question_id": "qst_0275::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a55032c4c2064671a4b0abf975438e1a", "dsid_17053c6bd4074e9cb526961d9a20b4b7", "dsid_9b01d799e6f94ec3a832b6bfcc02a4fd", "dsid_8cff0d2004fc4c58b2391a0e3cc71556", "dsid_31ea949a23ab406aa71df2e4ed140e62", "dsid_d97a869acd35489c8de7b0fd4c17b5c8", "dsid_896ccd2091734c0d8f3106063f32c377", "dsid_8c30aab9c2384348893cafbbdd68baa5", "dsid_b7cd82122b834b928ec615d788851bd2", "dsid_736800d2b1434921a2c5c19e652600fe" ] }, { "question_id": "qst_0317::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fb1829d221b94f98b219931b76903483", "dsid_ddc40a1dbc264cca9276a38359d4008d", "dsid_bd2988d48fc5483ca3d9445f2c4b5363", "dsid_c16066922b6742eea82a028cf7ed0769", "dsid_a10a64eb557643969aadcc433e634f33", "dsid_dce4aacbf8f44e95b9de70bd11589dba", "dsid_097a9f91ca894de7b7dc63d234ca5bf2", "dsid_759ee82b7f5044a49e263db5a325e8de", "dsid_fc88ae45e6ae4fbda01cba66ac0825ab", "dsid_cf3c1b184cce41eeb5acb92ad49b6ba1" ] }, { "question_id": "qst_0017::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c83811391709468fa9534e6495c205e3", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_54c2b31af18746adaa5b715e4e92df89", "dsid_a3f48a87dc254e8081bb6f451b4c77e6", "dsid_01f4ce1ac9ca4b85a471d39960e6be37", "dsid_ddd6993b770f422285f2927dbe450cb6", "dsid_fe5b2e04db744e708cdbf3e54ea78d8e", "dsid_5dc2e0cc2ac64fa0add42a1bf7baefcc", "dsid_374eabc0608142aaaab424e58851cb71", "dsid_bcd4d3e83c7b49e290da2552d2dd6f50" ] }, { "question_id": "qst_0048::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8412db85d95a45f7bbfb14dacedb4d8b", "dsid_88424ba4c44b4f70ac7902651a78ef1c", "dsid_23d9b80004ed423586a406dddccbe0aa", "dsid_f9e49403657647969208e34a0ebb6bb9", "dsid_e751c9e8106d4136aa3753006ad5fa31", "dsid_e7bb00f30b9e4d9287beb0b55e38910e", "dsid_bd4cfd315c2143a0a1199056b9bc35be", "dsid_a606f3c2c23f4c0b9895959a19f0141a", "dsid_64701ef21edf465081b28ae04db2aedd", "dsid_9f993e9910f24175a18123657ab7bc98" ] }, { "question_id": "qst_0060::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2e3e9c317bdd4811a56e079b1dabeb8f", "dsid_e5fe3fd0cf444668a64970ec00818f46", "dsid_9b30f5e9d84346fda6017553e8a78932", "dsid_b5aee2654e9a4f298d6d34212ed88211", "dsid_8b56f59b3a7744e3bb4b429912ad5aa2", "dsid_c2daaab889b246df8652445b3ff184fa", "dsid_328b258e0f664bb5aa545595343dc2e0", "dsid_538be95952e64be59c342d9d2c076382", "dsid_7bed0739a8f241219e4f3ab0304916bb", "dsid_14ab5ed61abf46faad167f1c56e16abc" ] }, { "question_id": "qst_0209::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_683dc2cf9bcb41b7be76949c6e0965e6", "dsid_c960c001e9f2479e9ff9567a2c1d795f", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_9fad67220fcc4e6596b8f2cc292872d1", "dsid_8eb182a4c2574d0fbeb595296e242087", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_71e27fa4eb0a4c90a4df75a404e905bf", "dsid_58a34b025e1c4ccf8c97dbc9e042548b", "dsid_cd6e4153efe34c19a4ceef954cba0fd4", "dsid_f4349b346c1849448221ba13ee2602d6" ] }, { "question_id": "qst_0310::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_484f118b24dd4bd3b290e59dce06e8f8", "dsid_5e768501ced8448fb09168b451b3e74f", "dsid_32c4c9ee86e44c799676b3ede42585e3", "dsid_a5d293df3be843caa748deb7c0946abf", "dsid_6c954a1745304c03b8614b4c9d924af2", "dsid_c7e2ac835af149158253734a0e7d8109", "dsid_9c63777c789c4f7db9e391024dd887ae", "dsid_6f46049065ed4efa914706430a70d3d1", "dsid_a9122972cf424e77985d6d519b9354d9", "dsid_eb3969b49ce444c092e6ef7aa1d10f55" ] }, { "question_id": "qst_0067::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3703253053174363ba6c47954166f7f5", "dsid_14a447cb5b1140709c7cb3cc9a88c430", "dsid_bdd6a5e832664f20aa0fcf81e1f80fd5", "dsid_16485be3dea549d984d3d3cd1d9da017", "dsid_84be9f4da0124b79b66051481cb10ac3", "dsid_ba24cefc5f9e44e8bb45cc2f3362ee66", "dsid_1f4169a43dea4f85be365b246ebe4ead", "dsid_02ed6e281d754fd3aa42eae826352c06", "dsid_5204d2a9973a49bbab1ea7a9fb129503", "dsid_2418f3a45a35495e8646639e2eb9c604" ] }, { "question_id": "qst_0002::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_03bea1fa6f944d479b97f6c30123c252", "dsid_a73b8219c8f14af88d36515e887c9d37", "dsid_1fb4ac4333314be3999767e281bb4a0a", "dsid_ac17673b4827489cbc0c55136fb48d8b", "dsid_f2d5557e5a66420ab69617b62d95bc79", "dsid_fd82d8ef7b0d4b55851e66bda26b9d62", "dsid_df696c771a704d58890d553ca946784e", "dsid_cdbfdca19bc54ba5a5400271b5f87d64", "dsid_2ea8e611a8b448ffab3bc5b927a3d466", "dsid_a6b710c3b5d442b080aa5527629a58dc" ] }, { "question_id": "qst_0350::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.6666666666666666, "retrieved_doc_ids": [ "dsid_f3e6edec6f4947a5bd5aa7fd557d7d2b", "dsid_939eea1a52f84393b0f87492b9aba8c2", "dsid_e2bf084a4d924d6d923a309234cfad2b", "dsid_fd3f961d8f6a4e39a6fbc039172061c0", "dsid_8590bba128564aa7a60be6067b6be779", "dsid_e47ee65f0bdc4000b70ee9f24a01a234", "dsid_13ee8fcce6874cdd842fdfa592edf899", "dsid_92241246d2c64816a5ef8f7fc925f7da", "dsid_7319adbdfeb94cf9a955a23092938026", "dsid_51ab2311e8e9430daf68a13516455234" ] }, { "question_id": "qst_0388::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8c15ff526e00491e9ccbde5ef4a4458c", "dsid_e34a086560eb4be0a95e3d566b810226", "dsid_1258e661701942a19c65972a3ad238c2", "dsid_f78be6e8423e47a497acece99aee2cac", "dsid_68fbf6106e38493e94a832e218fa14c1", "dsid_321f18280bec4e63a05203290726e89c", "dsid_53e7bb9a21d1477b88cf83e92cb434b4", "dsid_b2c5ab151b4d4407aada768a19611ed0", "dsid_63958405d90b49778a7367328516eacc", "dsid_a88e2249e19347c7a7c592ba75f8485c" ] }, { "question_id": "qst_0004::basic", "question_type": "basic", "rank": 5, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.2, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6c2cf8145ebf447ea475bd95c7354fd3", "dsid_7d9551aa21124d70b0e247c390d88622", "dsid_5e267e413c72438f8125a281ae7f9ad5", "dsid_45dafd73b0704a759d31eb27461b13a1", "dsid_6c4c1c875e704f09b4d791d64d7bc7e5", "dsid_41ae740164754c88b36a015e63ee42a9", "dsid_f29b67bf611e4a39a17c611cbbdcf3fa", "dsid_719ea7c102a945388007bd2094259314", "dsid_e7dd8aa445994a6382c0ebeac6151bef", "dsid_3d11f11c166148b5b26dee04563049ec" ] }, { "question_id": "qst_0344::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.75, "retrieved_doc_ids": [ "dsid_4b2ac6984bdf4737a18520e158b04453", "dsid_409618a3958944f9b78362ed7e01bb33", "dsid_8c704516b3e94f9fa987d242dd100647", "dsid_ad6beda0999448ada51637be6063cac2", "dsid_a5a5235d973e421c870e11ceea4887ed", "dsid_401fe66f82e2440f8a5eb6f64cbb9de9", "dsid_ef048583ee2a41ceaf97142a4ad3550c", "dsid_51895b7e9f2942898291b66b72886b15", "dsid_6aff4d7b82fe48c1976591d17ac3aef9", "dsid_3809baec7ccb45219ca7bbb223ae8dc4" ] }, { "question_id": "qst_0170::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_9fad67220fcc4e6596b8f2cc292872d1", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_b1287d634ce84d20a643e2d20af168a9", "dsid_3c69e5d0b7a6462d9c7d54f40dbb914d", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_11279679cf514673b356a29c79b8dfa9", "dsid_5ab083d7805847558e9ae08cf9fdb267", "dsid_3bda11c47af140e891575b02c7c9433f", "dsid_a032658b571d4562a1a98485c5adba4d", "dsid_13e6334193cb4c84a85546760631dc5b" ] }, { "question_id": "qst_0213::semantic", "question_type": "semantic", "rank": 7, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.14285714285714285, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_73f607f4ddd54243bb196e7b1393b329", "dsid_2c33f36bb9254f6cac917de0f81bdc60", "dsid_f85721e1c048491e8222a383654709ad", "dsid_f9266d0ba58e4d51b088a831a3f230a6", "dsid_f560ef99b8a44495b9cdd9154129819c", "dsid_4b07ee432d54461e8e41640a184a47ef", "dsid_ad14c3cb06c4494093dfcfff5098d89d", "dsid_789a344b1b094259b1ea5e861d02a316", "dsid_4e0aea2ee3134b759f7f839796a3d548", "dsid_bf494e67fd144f43abdfe0d7180d80ee" ] }, { "question_id": "qst_0368::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_abfb68dde21f4fd2aa7d70c472447f89", "dsid_16003ee4fada406da751b9bedb93d5ce", "dsid_8761a5900a7f422e8a2606576740f2bc", "dsid_826a15051f844d6ab1c1ae9bf0b49bf2", "dsid_5ddf1c80a8514569a961768697429dda", "dsid_3f2cf90e674341fda45c81220ab93eda", "dsid_94b0d0306935482d820e38d0bd986139", "dsid_18ac88b8d1ce4470a99abeebfca95cd4", "dsid_94133024fb674d94b27722aad6468c70", "dsid_a0f8a0c7eee340db95e56c449dcdad56" ] }, { "question_id": "qst_0056::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2cfe3f56de90412599f3c7fb1b1df829", "dsid_9229aa41dea84edeb3357de2145bd30d", "dsid_a8029a2cf070490385732a8136924157", "dsid_4f82a120ef1442fa9bd7189c9833e3ae", "dsid_821022eec9ff4aafbbbb9707ccbfd7c9", "dsid_8d35a28762b4429b9869b7370d3ab332", "dsid_fcaaf037e4394b9a8df2c679d6dac081", "dsid_e41fdaeaf6294821872674be79622456", "dsid_045f0fda65374286be9a414c3fd157ce", "dsid_bb4f8454afdc42c78133a50dc4cece2a" ] }, { "question_id": "qst_0071::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_e21dd8168db644bba2c6db2e9cf86fe4", "dsid_d9e7c0e0d2614a9ca91a87f83e6958e9", "dsid_4affe9870ba54d988c229686a7f819ac", "dsid_84e1e2540fed4e7da632c7644d822cd2", "dsid_23b34ed3208c4d0f8ba1f8e23c8fc145", "dsid_5df06600c32b4cff85b5a41988d0dd49", "dsid_e6fd347a958d4150bbfc96aa1ed9bf77", "dsid_bab5a332825f44e7a796882e7a7dbff9", "dsid_a6838348ea52413e86c85dbe5398a7ca", "dsid_d11427550fd14a8fb8335ef28fca5176" ] }, { "question_id": "qst_0116::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_6c57c9d4e08947d6bdd35a0eb9bb7c6d", "dsid_e51021765c694a65bd60b3d38b12db6e", "dsid_3c1d24c5218f4cfdb12df85e45d3ea7d", "dsid_d8ec0649c34449d3bc48f7a43cf62372", "dsid_c356d1d687b4493f92c01b693d4e98f2", "dsid_b6fb2620d47d44e3b17b16b42f1e021f", "dsid_d1a5894a308e4737a4c1a591e962d337", "dsid_ad64590bc0c049a3bc525680bd367f36", "dsid_0032ae6f295b4845b21638f56eae59a2", "dsid_69216a68434247baaaadb717e80636d4" ] }, { "question_id": "qst_0227::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_8dfe3e03d22d4e61b79b11b6012cc249", "dsid_46b0bc1e97ae4d8792d08e6f4122b948", "dsid_46383d57889e429380b4ea4d9bfb5646", "dsid_404e5c381d784cf09e2506e993abb359", "dsid_3c8bb2d1caed4ca2a6c3d289175f34eb", "dsid_48147d8287d04e928ecaf0bd7b35859a", "dsid_76e88c3d7b6249319af62f5d376e749a", "dsid_91b0d3d9ad6a4323ba843719e2cb83be", "dsid_8ed07df6c3b7411d94410c1584f67a7b" ] }, { "question_id": "qst_0019::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e728a420a02a4ddcb125e6adb09d211e", "dsid_eb415ca0c9ac49208ca007c2ff7c3793", "dsid_c5c60dcb10a941f1a02ea21fdf708e46", "dsid_39923eb26df4401ba20689b392439695", "dsid_5d35e2c0250641f58eb04a8dc9020282", "dsid_f38fa8a77d194f6bb32d3045fe06a80d", "dsid_775221c79f314bcaba5bfe12df57479f", "dsid_b6921e85d3ac4bb3aaaa91c881b38628", "dsid_40774fbb68b14c5e85e20b9648e1d347", "dsid_c6ea85c837114ce691ac1261433bd9cf" ] }, { "question_id": "qst_0047::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7a2cfe4b7e574e6995f33e64bbdbf426", "dsid_b0ad9c8fcb6b492a9e5b49ea4a88f557", "dsid_cf989b3506d1425aa75fb226b39eeb31", "dsid_7ab12a0c235947b4a69264e27b2dcf07", "dsid_847e5271edc54279987c9e0ed2c3e11c", "dsid_944f4416637044c9903ca2f0aa6d3395", "dsid_4c612d0feb714d2881fba0774f4f4a3c", "dsid_92f56ed27b75455f82f43528f6644f3c", "dsid_7a25e9b6949b4b7da4f050661f626990", "dsid_dbf76e715daa4e6eb90d11770a2a2f74" ] }, { "question_id": "qst_0101::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_11907800370d48f2afd31607f3dfcd96", "dsid_c4b7fd8151c94951bca269fee10d454d", "dsid_5b2a860fc29047b4b4f68d0862199bf6", "dsid_d9147b6f83154ef38f4816311a94d63f", "dsid_0d92eded0eb445d58f4ae40a067fc3ec", "dsid_fb6eb53e8969406d844868b30ac46b20", "dsid_1b07e333fd5b4858a99e0aa3a1b0914d", "dsid_92bdff7872b0455897e7d83511bf294a", "dsid_ceeedcc62a1244dcacad865529e1985d", "dsid_4559491f7eab41aabf2578f290584bab" ] }, { "question_id": "qst_0079::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9776bd75ffbb4ca7b10669be1512daa3", "dsid_94981f0fdf8448e2bd45ef15ad923597", "dsid_5826306ad74842578a6b217f732361ad", "dsid_41ae740164754c88b36a015e63ee42a9", "dsid_d3295d19ddb2451e99da9a83d02a249e", "dsid_4cc8a3bb41464b57bdff61fdccd9eb97", "dsid_3721dfabc1c1444980bc75a6ce5a4468", "dsid_d5bcf6fb194243e6b1eeef58d72d5a47", "dsid_707bb2a2dee54adc92bae423c1ac3189", "dsid_929a39a9c8f44ce88bf98ac46669f00c" ] }, { "question_id": "qst_0426::conflicting_info", "question_type": "conflicting_info", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a6a4d42e97ce47aa9509e779f67a6463", "dsid_2bb75fde3be24cfcbb35ecb1c14dab21", "dsid_c90bd544f3b5437fba711492db31aeea", "dsid_553e1651575a416dae6b92d019d2f9f4", "dsid_83effe9968594d25a5ecb719178034c6", "dsid_fb702e56d43d4015837c911737e4d51e", "dsid_2680a894a35b4117a2220bb308c4dc23", "dsid_755e9423ba62444f908dc56936de4c73", "dsid_10c7abea379041cdbd9ac9c7e20c299e", "dsid_f7facb7c473048a9827ed6db6dbc71cf" ] }, { "question_id": "qst_0327::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_69e745b76aa34eab8721ba0c88482ec0", "dsid_3437967651da43819111a30e7bed89a1", "dsid_0a6fd9644aa24ea7a2e62b86d712c4f8", "dsid_c52c609c0cdf4211a7e8c57bb1da72e4", "dsid_389bf7ec1bf54db5913767be3eeb537f", "dsid_ebf57e8846c14360a7bc803076cbd03b", "dsid_c3f5e058c55c4cbb8ae27bf304598af0", "dsid_0716297857b94378b332d014fa346174", "dsid_1cdeab0ff7e54b34a0f7e3b28d91427e", "dsid_0e629bc04fe94eee8dac7219b1fed0f9" ] }, { "question_id": "qst_0461::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_22b0987227cd4fdb94d67e5a62bbde40", "dsid_0fa69b4e576f417ab759af8e33cb049f", "dsid_3649becd3bd14f4f8bb788601f02fe46", "dsid_a67adaad749f4570a28508abf05d8a5f", "dsid_c5288dd4874345adb80f4a71c9a18773", "dsid_6ae2bb84d2cd44bd938bfd39988118fe", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_c54fda6423f24bd7996f2c79baa56cb0", "dsid_24a582276f714e568ac5f5e20862bba9", "dsid_047e2ba9fd4e42d09be10391a6448d90" ] }, { "question_id": "qst_0022::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_20f19570273148f28305253a62c95fc9", "dsid_59ccf7ad27ff42fc9d3dcfa619985fea", "dsid_53098bda08dd4ebe9875d92f6f18aee1", "dsid_6627d1d62b60478197902970db4b3fa7", "dsid_0f1e6ba98bc8489e953a2e95ba6e368d", "dsid_7c6f649b898746079dcd66302c50f8f8", "dsid_c37c979e60644e5b989d14a3259f0b68", "dsid_6cedadc2d119440daa0207b31c73f32a", "dsid_6cebde3a79754ca08d33abc2ce64f71f", "dsid_14a55465fe5a45f7b3ed9739c015c682" ] }, { "question_id": "qst_0103::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_e88dd0c5a63f4fe8bd73783ce44a05f8", "dsid_911c14a0c2f54a27b9d7474d73c82aac", "dsid_248aab7eb03d4a3c9c581f07a573b6d9", "dsid_f86330ecd067401c822845816231f124", "dsid_5a4540141c924a87bca649878a051c49", "dsid_27cf84eb34214fce8304d542bb11228b", "dsid_50134686028e4b73bce2a121571f5e99", "dsid_91249de330b34aad8423c8e7b60e37a6", "dsid_16fee312c4c64e288373bb3cce935fe8", "dsid_43d222376f6547e7a825cda385a87234" ] }, { "question_id": "qst_0234::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b16eb801dbc741558b841c7c3047e98d", "dsid_512c80bd4c654348b294c8a63c2f0c9c", "dsid_4ecc265a59c5499da505794b95399398", "dsid_c2c983ffac43449a9d663472f0438484", "dsid_7f923b8f83434725b5c52e2b2a436a06", "dsid_6116bb8f6eb0459aa4ccbe32800844a1", "dsid_f6e057861b994223a82a13a56c715e5d", "dsid_bd78dca6e1634d2d984e92eb914da5e0", "dsid_26b47ece45a04da0981997c7f29a6fc8", "dsid_9d0d9dc7e1d84933b9f89376184b0142" ] }, { "question_id": "qst_0161::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3b35497fd1e448d2940dd075c1d68bc3", "dsid_2fb08b7571444bdf894607549642498f", "dsid_726bb419a3d84d72b73fd163ecec8036", "dsid_8f8d7ac4321e4d38aacaed9f44cac1ad", "dsid_bd18e7244b064950ab6542ac457b5b66", "dsid_5888e4cbab624cf3b21975485b81a535", "dsid_c896f88ea22041caa48245eb765cf988", "dsid_6486f4b831984dccb68c2e3cfc9473cf", "dsid_2deb330e657b4009a56d4756ea3cb0e5", "dsid_a3dded417a234c7bb880ac061180da92" ] }, { "question_id": "qst_0069::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d873f1adbe654259a48b9ba497e7e4ef", "dsid_d74e611409784ae898c11068cd02cd55", "dsid_de9c1d389e964346a5d863550ba817a2", "dsid_d0660866e36b4877b1f828b395181c9c", "dsid_3237666f5dd74914b1bb5cc5dd7630b2", "dsid_dd5f0e9d5435414791786f0ab4cd0363", "dsid_893630e496774ef28079a910a82dfd16", "dsid_3f710f583a1f40ba95ef2c577641f88a", "dsid_0b17061247644846838334f98b738f9b", "dsid_acd812cda19c40fa8c5a9c044aacd6d6" ] }, { "question_id": "qst_0145::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_182c34ee62c54118b48c1390e77cd3ba", "dsid_f1ceaad5f8c74d578f5f926ea3c3cc63", "dsid_25d772e911ce425094f28f022f314d22", "dsid_e47c10f5903d46b2a1cd18a55b13f205", "dsid_d656c22e43034e2aaa89072fe1efa4b0", "dsid_56f5574cdcfd4f829097cfb8cb197d3b", "dsid_9f1ef10c8f1240fba7b18068815b6bf2", "dsid_5a7a71032ce149e1b9979bf18bacd6f3", "dsid_a110cebd6e8844afbb800d73e451bf55", "dsid_c111070e85aa474eba532c2599f9dc3b" ] }, { "question_id": "qst_0439::completeness", "question_type": "completeness", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2cfe3f56de90412599f3c7fb1b1df829", "dsid_6c9ffcf5dfd34fcea647ec17fb7a591d", "dsid_3b7139962b28486aaa3a498cb375b2a9", "dsid_045f0fda65374286be9a414c3fd157ce", "dsid_6483467a98ed4097b79676e8cf447f13", "dsid_6d87679242a842b08571001f0b3d2dc1", "dsid_8d990a0d64f94f8ca7e7421bc2d5549a", "dsid_d9381f8b97e24d8f83f3c2ef32d21a65", "dsid_12b40ee174cd4ba2abb680c627e94bbe", "dsid_de5e53ab67674b2ba9f80754a4cbf835" ] }, { "question_id": "qst_0027::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e4b8a2a6b6594d77b1880327d506619d", "dsid_13b17d1c09014c60a7fba5510ff0de37", "dsid_f309b7f44b344bb9ac66d5e552273292", "dsid_6d3921f9d2444e76b93a3ff9ffd81984", "dsid_dff824f5d9fa409c965fbefb1e848123", "dsid_4a83331fa3d540c48e96d913efe857d0", "dsid_c1aa31b0c6c34645955d7f8bdd622705", "dsid_1ce2d7a1ddd749f7954bc2c8362200ea", "dsid_77ee049c8b054dffa5fea08649bf50e2", "dsid_8e9d8eff077e46a2b0e75d3e66f90f69" ] }, { "question_id": "qst_0418::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b837e1140d1a4c3d8efd2f42d9b6a799", "dsid_aff78a10eb974cfb99cd820f9ac672a6", "dsid_b01c12d7b75240c6a712313d764bd8a7", "dsid_294d72f1c7f54fc1bd8f54b542d56c5d", "dsid_6ddbb46c266647f4922b8107fd6ea395", "dsid_981f4a0281054f9aaaec7887156eebc4", "dsid_8678af84bf4d4f4f8e796ac2208185d7", "dsid_26fe354d38864e5ca426dabec101045a", "dsid_c0ecd55042c8494a86c42d8b8e02ab60", "dsid_9cb816d6c07146698ecc2e52ecc8f2b6" ] }, { "question_id": "qst_0024::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_74ebb73359364bc791707a92a3227e3a", "dsid_cf06e75be4f94156ac338f62e3ee028b", "dsid_e70afd2743e1460289bc278c150b61dd", "dsid_90c8aa47b95e4a15855e353dfd4b4fab", "dsid_558b3331c26c4f99ab4af3fcfff4f9e5", "dsid_40ea171d24194791a30fc1877c028dbe", "dsid_312f8b8dfba74be98c892d2cebe9f068", "dsid_00908a2dda4b4d359194a091019e8367", "dsid_58c500179d664087ae073aca9e418d27", "dsid_546e110c14144ff9a1a02d9145c43177" ] }, { "question_id": "qst_0339::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 5, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.2, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2c7df9209ebe47ba982b9382e102d292", "dsid_9cd367b225214f48a77dc58684edf6c3", "dsid_911dbe96d7ae41a2bf297fe997c71d54", "dsid_51219e8733564137a709b7a5642f960b", "dsid_81208556437644a0a4dedf168baf81b5", "dsid_14fe6bfd17ad4da8a1b22c4aa3bf3cfd", "dsid_07460d87fb8f4d37861a07fd1dfe0bb5", "dsid_82720922468745d283cd921b6d92a2ce", "dsid_a2eb1ba41a274c16961aa700438fed1f", "dsid_7e45b8647c024c439fe4ee7bd473dbf8" ] }, { "question_id": "qst_0285::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d6ed5dd197bc4198b6a044c30728217f", "dsid_a3ebe69f7f424a6a96a15c7b7b1b31b5", "dsid_32ff7b92df1845ffb056e8a32c46c556", "dsid_e62bfcaabcc84f3eaf44c7f6bb1219dc", "dsid_121510979dde48b2b1607d9d8e11f5bb", "dsid_06c480be9d4a4da9a51aa138c658d549", "dsid_431c5eae534d422d8c3af6cd5ad0cd83", "dsid_940e31a965fc4916bdcd13df2f56a611", "dsid_736dd88268f1407088b3aba9d09bd469", "dsid_782f16eb2e50458bb62c35dc21c0702b" ] }, { "question_id": "qst_0091::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_da1a96a29af44f19ae6fae65f4a87ec6", "dsid_9eda2ec9cbda4188936246d888df1d0b", "dsid_0d359b15d28a4b87b7d371b30091732d", "dsid_a3481dcfc3554616a0b252b5fd6411bf", "dsid_610d2c74a60441e886fcc39e351a4176", "dsid_48eeb6667dc24b10b3d6368c5b53a8eb", "dsid_d947fa3ac4444c8ca7f9cf2d50a1a2f8", "dsid_00df56b3a60a4cf9982141ee4852d450", "dsid_a0fc5dfee62543ca8365bb43798130aa", "dsid_a463391ac6a348e3a5e729f830b359a0" ] }, { "question_id": "qst_0075::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8c60fea2755245de913a78b2e29c617e", "dsid_11ec526dc14943b4b47b591dc9f47b28", "dsid_36b163d43c51450fae48cb0795d7cf22", "dsid_5ea82211a864441c8f847c690182d761", "dsid_a0de5e8efac04b8cb8b22b56ec11bad2", "dsid_5a9887faefa8440a8abb22fa3b600631", "dsid_13811764ba9f4cb792f4fab84fbfc546", "dsid_0d1601c33f984a12b88bcec83887ba15", "dsid_14f75fd67b604df388d7d0a17b012a33", "dsid_96f6f8abd52a4b31af02bd9f7b92fa8d" ] }, { "question_id": "qst_0470::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_aea4790d4bcc45859208d7d705f682a1", "dsid_27fe1b2619a44143bbf15407aed92961", "dsid_751f6fb0aa5a4d1dacd6ab613b3a5aec", "dsid_5fa24d05c02045afb2aac4e64e65640e", "dsid_eef6a2e56beb486fb0c0436372eeeec1", "dsid_e165a26030b64440a1d6eb3c647436e6", "dsid_a8f6d626244b4b79b5cfb3163a4098af", "dsid_ea6d8a686660485c8e570a8fcc96d598", "dsid_8c0cfb1b333c4d2f9590209f14c2b816", "dsid_bb119bb2dcab4ae098996b49ce810faf" ] }, { "question_id": "qst_0084::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ea23b1139d4141c483f46443089031e0", "dsid_dd20fbf1be174b9f8498dbb81ac84611", "dsid_f2f77dfb5bfc48e2840f8e023be3aba8", "dsid_0c0fa8b46ebb4223bb4511676c201832", "dsid_6011743a83dd47a7ad754f87cd587b01", "dsid_43cd51c13a204e829fbb3baee7748191", "dsid_b3aabf9354844c7ba23b11f7d7c9e60f", "dsid_efcaec900d654ab3a6e5b71fe0e47b76", "dsid_ada4ac91a8634400b2cf229a1372ca44", "dsid_d44037462cb8495fb8b6e170e98c8a71" ] }, { "question_id": "qst_0043::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_795498b20e9c4d38883c8c46ab3336b4", "dsid_a0de5e8efac04b8cb8b22b56ec11bad2", "dsid_e7e0bb32eb8b44b8bd6bb8a4e9071d08", "dsid_aabaaddfdbf040c4ab5534b1158edbdc", "dsid_eda10a8a384647768f1a30a51e7c81c8", "dsid_67bea99d2c424fc5b573aad68fca2c7a", "dsid_33ccd055006a4937b0ce9b375f3c87a3", "dsid_e4e5f7ec247b43efb0a19713cdc32c3b", "dsid_13811764ba9f4cb792f4fab84fbfc546", "dsid_2c0b893a4a704348ad6ebfe0f0ccf987" ] }, { "question_id": "qst_0422::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c5f0f39c0d2e430680b1b12b523493a4", "dsid_4a2476f041294cd6a9fc0344db20a553", "dsid_d417b614467040e0b9f40ab37dece165", "dsid_73c16e3b43a348c7bbd440c86a40e5ee", "dsid_caaa763133944643b120dd51469feab2", "dsid_dea9de3e5e904e4aa4e173e254e785c7", "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_b2f61ad69b4b4f30b258e32415bfe6e6", "dsid_f59a0860f18c49acab904cea19b3a610", "dsid_56a1559c43db402b80bbbd0a5588337c" ] }, { "question_id": "qst_0094::metadata", "question_type": "metadata", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b2aedf3ac1934dad9bd4d612b22b58e3", "dsid_f8ce92dd6d4941dc9d8f8f24e3dd9d45", "dsid_746c88d9520549c9b142985f3af6974a", "dsid_f7a0934d387d4b129077a8515e60801c", "dsid_0f65f6c989dd44798aff739ac683b544", "dsid_aacd32b93d4948ef815dcb85d605fb09", "dsid_bd8c15b5fb3f4007aebef0045e6257ad", "dsid_00c123f8180e4a05b3e53d65154ef195", "dsid_f62353f2eb2f4ed9ac509107ee1ba9bd", "dsid_869a1aa8c76a4a07934b712ff883302f" ] }, { "question_id": "qst_0081::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7a7c04eebb914edab49f734c8e814beb", "dsid_09096ac3d6014989a8c8b54e4db58043", "dsid_3a0b21ec74924d81897740b3e8729e13", "dsid_044413760e0b413bb426998825cfa99a", "dsid_74fb68c944f04098afb717419b8601ff", "dsid_5dc088ed299743399af322d61edc8ba8", "dsid_16f658af03264475947f954c77a925e0", "dsid_cfaf7a886ffa40848a446c6ead1a9442", "dsid_8b49fdb4f1d3439290449982939b07e6", "dsid_4c7354cdd8f64160b67b73185ba19328" ] }, { "question_id": "qst_0068::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a3ff6f6eec4b44f69c0e88303c264c65", "dsid_58a34b025e1c4ccf8c97dbc9e042548b", "dsid_71e27fa4eb0a4c90a4df75a404e905bf", "dsid_45b99ac2827a42c1ad6f01d17b6944e5", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_0e15e91cd86f47d2987e5bd8cda76995", "dsid_5faf1e6f0db843668a4922cbf045f677", "dsid_e6f4d2efb6ab4fa0827a09b203e5e1e5", "dsid_a42003246adf4a08a787bb8a5869167d", "dsid_f88c8871ad14480cb3ac09ea3dea9846" ] }, { "question_id": "qst_0107::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_509b34ee9a334f68a693e61dfbf558c4", "dsid_566f5beae5304a76b92f2bd850a5577a", "dsid_342b24e0e85f4171b8689680e455c21a", "dsid_c31284f73f7947fcb0ff58196abe90f8", "dsid_3e2f413f5c424a2a80fd1cc48fa302cf", "dsid_81bdb0fd1e1b4604a5f43d6b6a658c52", "dsid_1721e73e7955467b8455753aa565efbe", "dsid_d72d3839ac3f4e0380ed031392f3bb23", "dsid_a5efa7cc513a46a08108a0d0225b7d76", "dsid_f8a5c2f201e146feb6e8d2aef7a615ec" ] }, { "question_id": "qst_0079::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f63e4d18ec674e96ba30139d0aa7c50d", "dsid_310e387ea02844a589d5b79d00141d74", "dsid_ad64590bc0c049a3bc525680bd367f36", "dsid_eb9be5671f2d4d798976c0577663f8c0", "dsid_1deb78038a714d44a449756bb96b4f9d", "dsid_a2c8365b6b1a476a9ef897e716a25621", "dsid_1da057b4f1fc456e972653530a130d8d", "dsid_a490721504e14e09b5516405fb57776d", "dsid_f045206c409044fa95dc6175ef538ee2", "dsid_409b0607f402469abe58a56a37e2d3b8" ] }, { "question_id": "qst_0076::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b097ab5d9e4c41609f47432dfcb1508b", "dsid_b3a664d939dd4aa4802ac81cdf9896b6", "dsid_8e85c02011ff4ddc92ff86bcfa3cb72d", "dsid_bebef79549ca4c2792bb73bd30e686dc", "dsid_98adce48b4074afdb2425dab5ffeab26", "dsid_4c400c2f6b6d431298528b4459edff6a", "dsid_c5b8c516b9a7423bb2db89bac98004ac", "dsid_19d5d2f6a4cc40cfaa6086ca39b2b741", "dsid_24a105510e3343cfa11f6794dfcffc57", "dsid_bb999759c2094c5684d2f13a4dac05ee" ] }, { "question_id": "qst_0413::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6df52fdb96ae4edcb76464738bca3340", "dsid_fe3b67365877461d82634b38cae7601e", "dsid_9e5644aaa8fb4c34ad434aeb0f333fe8", "dsid_94981f0fdf8448e2bd45ef15ad923597", "dsid_845172d9dfc84fb1ac13457ca486f673", "dsid_513bf29a04584b6aba3f0c9084ec653f", "dsid_a5f013d6264f41d48c47ebada02ea659", "dsid_708328d8d8d9401aa41e36b109153952", "dsid_3f0528402e794212b7208285ad3e09f2", "dsid_562eec17e7d945088abde66e1a0575cf" ] }, { "question_id": "qst_0363::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_b993fa19b3d44382b709e3107ab1ba77", "dsid_a820a561907547688422879d2dab3e37", "dsid_9a2f440e8f7e4801b76233767fcef945", "dsid_f81494a2de6f41a6a36110a07db697b2", "dsid_f2d5557e5a66420ab69617b62d95bc79", "dsid_44b11e967a5f4029aebb720891f66450", "dsid_130e02b5f9aa4ac78be9f0bb9b648f72", "dsid_164e247cc4874908a685434edd428979", "dsid_6933f9241ef140da9d3bf98be52be867", "dsid_7f52d4c78e8f4708a11b9ba1301dcb6e" ] }, { "question_id": "qst_0149::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_178ff503c6fc4a4db593eace57e6938a", "dsid_42c3e2d70229415d82cac2f9b795bf1d", "dsid_d3825457e10041d5bdff57cb8eea5085", "dsid_39606fc7f6bd4b98aa9821b31d057bbf", "dsid_469a32a1152e41369559c906832da1be", "dsid_e07f90d747da4e2f9dadfcf0575439f8", "dsid_f869eb7c3a99487e977b8e63191e0227", "dsid_bde79cafea9247cb9f4ed32426ae0bd6", "dsid_f44087547d5342d89251d9298fbd6d3d", "dsid_6f19a5969eb14cd4804c5ae73d7bd3f0" ] }, { "question_id": "qst_0132::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d8a18c9f03de414faeacfda3ad53e315", "dsid_efefc1f7e55645808410b3338a3d5200", "dsid_854a9ca8daf54716b75c22377df89654", "dsid_5f16fc64309c4a2680840e3f6991b327", "dsid_02e9beaec3dc4f579c3dc8c892a8b05a", "dsid_863667bc41cf499bb5b430eaaf41693e", "dsid_b11d65828e2c4f379452c7698c8c28d5", "dsid_c40b65a75c4c46878b68994d5c68bfe2", "dsid_bf15bb68f59e4ec48ddb51084eb9e10d", "dsid_87892aad3aba4913aabfd2a12a1ebdee" ] }, { "question_id": "qst_0396::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f90b68cdda834c6fb4c76508d359c72f", "dsid_f940e7c95688406cb497b0b1dab44e99", "dsid_26db0d269ce04474afdae8bf7265e1ca", "dsid_00f927a59bb142b897f2f1d200325a89", "dsid_c3b14485712a448d8227f6a1efb3cbb7", "dsid_58f2cc3e008a4060bf5f9fa2677cb20f", "dsid_a5cc666385c242f2a37a57206cddace2", "dsid_f0b14f9b0cf2409baf9ee85b61b507c7", "dsid_d57c1953e54748f9802ee76d98a410a7", "dsid_d332c40495b745a48087d2f0349c90db" ] }, { "question_id": "qst_0435::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ce60931a1aea428b8186228ee40bb497", "dsid_a134a3fd3c244a52a8d3e1ff515fdf58", "dsid_431b0713c42847bbac15f3354776db68", "dsid_03e22d519f9b40a5ac61a03e5c4620d3", "dsid_8116cb3e580543d7bde1c57df351a3f3", "dsid_7fc490f8522b486c8f4b665d5189066c", "dsid_76ce1fdd2b4d40d185ea0b5b27ae9d78", "dsid_b74460c600c24e9890460374aa27622d", "dsid_879b30745b564e9bbd2e8138139bcd15", "dsid_1620ac9de7cf4035bbdee559c3596d1a" ] }, { "question_id": "qst_0082::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b538d114ffc44503bc41c5ccf49f8fd5", "dsid_da0ca8d5d0bf4951bab18eb805d49215", "dsid_f0308bde052448f48d9dbc37d163c983", "dsid_0257bdbb401e4b0a80aff76f57ca7ac7", "dsid_e630006e7945458ea50b048c24a744bc", "dsid_b4f5d647ef7c44389b4052782c959bbf", "dsid_0c504258cd0c432d898e6711fb90eaf6", "dsid_1c8db9b0941448b7b6a5389987716df1", "dsid_2c32c59025364bdcab66e76b214faf60", "dsid_94105c439082474ca4fd036706ae204c" ] }, { "question_id": "qst_0083::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b105603f46224f539cb8a1b93c9aed21", "dsid_3b9483f629fd423d81ac49a7efff89c9", "dsid_946e4cd90985424aaf4a7536571cdaa9", "dsid_c3e9f8e3807d4b30ab782efbcc2e45d0", "dsid_a690b86aeaa84867be2190c36b1da63e", "dsid_f94eddd70c7e4a028e16c760809e9333", "dsid_8a4fc00a0d7e4957b2682914f9c21cd2", "dsid_83259ffa83a546a3adbffce5381833af", "dsid_7de214c03c674ce1bab60fd9cd74dabf", "dsid_d171e3e2d2b64dc686f4558085947647" ] }, { "question_id": "qst_0037::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c1a652436b2e4c949f5de9f5da227cd7", "dsid_e0c3901033834220b87974050eb7daea", "dsid_945ad52b9ae04034831d4da0e9d091bc", "dsid_3532bf9d2d334253ac90846ced71bf1a", "dsid_3c79ace6733f476bb54d1d289894d09c", "dsid_06553e3df89f4e36bc6383ef75eb1619", "dsid_e9ed987738a34badbfc189ac5c8eec60", "dsid_484f118b24dd4bd3b290e59dce06e8f8", "dsid_69257177a42348e0b83a9ed1d57c67d9", "dsid_c9726477abec42dfa800a253d02be5f2" ] }, { "question_id": "qst_0241::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_586b30ff5b3045948083a2b9d2802403", "dsid_dd5a1af182ea4dd38ec5e9de375d9904", "dsid_af0820a88cfb4b9fb8881e72cdadda5d", "dsid_c47682a5eada448692c8dfb418c9cbeb", "dsid_5e9f6ee1760a48ab87f5b6d7f3b3d42f", "dsid_200c0d311412478dab2d58fc1957a761", "dsid_745f0dc98bc741ebba4e17d0a52c57ab", "dsid_22a2141e89ce4565abd22fe7d5cb2103", "dsid_3e6b06f54c7f4627b3758acf62d00b1b", "dsid_bfecd279e6f64d928fc2be2e33882aff" ] }, { "question_id": "qst_0055::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d7aeb7de2b6c4ae88f46f4b0e4f2aacb", "dsid_d34dbe143a1e4af4acf8cd6624ff01b8", "dsid_3d589feb625e402e96b9b17e8fa77c27", "dsid_ba6c192eb8f14f4699cf00d85462a8aa", "dsid_2e3e9c317bdd4811a56e079b1dabeb8f", "dsid_1e9f3309e6904a96a0a7a031336a91b7", "dsid_02af32bd56614818b61e5263158f1381", "dsid_582cf753660344acb9d061d282499d4b", "dsid_c65e75df3dc349d48f2874d5179eda55", "dsid_748d055a6f2142de8a28660f318e27b7" ] }, { "question_id": "qst_0282::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_045f0fda65374286be9a414c3fd157ce", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_a52a2354f2ef4a26be3e96dc2f9d6a63", "dsid_2cfe3f56de90412599f3c7fb1b1df829", "dsid_6cc25761991248b2b681ccef319a7588", "dsid_bdc205d1541e4e61a7a4b2f70c3f5583", "dsid_6483467a98ed4097b79676e8cf447f13", "dsid_a5612e5fe92a4134a1564977a1e45755", "dsid_faf3246a6ce845e6b5df0ae52f439399", "dsid_b0b9878c7b11452e9c49e55e4b0b2b11" ] }, { "question_id": "qst_0065::metadata", "question_type": "metadata", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_09c9c54115d943eda12888709ed23695", "dsid_7e82d2ac51114ca2b1c33ce55a96e499", "dsid_abe7a4bcfe2147eca3a3afc5287c3949", "dsid_f2a514f3308b4445a3e86b9839eeb7d8", "dsid_0e4be49a69644f43b9f8b2b0c1b95d38", "dsid_42cd7c46f7854112a526ec0a8b1ae46b", "dsid_592649dd830742ad905fa996fc9afa0e", "dsid_c2af52032dea4ca39459d2281febe9ea", "dsid_f356c00f7afc49d2aa2639247eaf2c44", "dsid_5016f308bc4f42f3b9f85c90e40c5b9c" ] }, { "question_id": "qst_0036::basic", "question_type": "basic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a195b8bdde7a4f49b462b1ad2d2bdfc9", "dsid_3e2b9934184e4cb99c388c50f90d7e36", "dsid_c239203a0c7d433f8079475beb34b371", "dsid_b8cf75b6e0c748e5b9ae0fa3eeeb8c32", "dsid_3301f27e480a43f587e0601093e63e1e", "dsid_f94b93a4bc7948ddab45c371187dc6c6", "dsid_6b53af8a237b4979a1092134eb15c428", "dsid_e2bfd0c716374c88ae9e4af6040d0f28", "dsid_1394f4a6cf644d1f819b17dd47270bad", "dsid_0212cde606024a35bcd8669924657cd1" ] }, { "question_id": "qst_0297::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ef4d0f403797422ab9de5e589431921c", "dsid_5367ff2e8f674d2798ea137a5e060f8b", "dsid_7c36f49e191e42f3ad2bf450391c2eac", "dsid_a150825e326648cba900845e8faf6ec0", "dsid_d5aa0ebab2d9498d8e2c065b09dae62a", "dsid_f9f1fbf4df8944c68cf030f932aab2dc", "dsid_50a004c55eec403eaddc2fab715e78e4", "dsid_a3f94972391244f38ef773c05926af83", "dsid_c824b16a05a84dd7b1f610ba6094d96d", "dsid_9872881b3dd64551961fa390fb7d4475" ] }, { "question_id": "qst_0248::semantic", "question_type": "semantic", "rank": 10, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.1, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_08a352047d7a43ca836dabb7dd50787c", "dsid_462f37885ea5406284727925ca5ded7d", "dsid_7c52f59a57b9405a9fd26c83d43bd2f9", "dsid_f51fe807cdc749be8226d3867de81da9", "dsid_8bee5f2c68b849b1b58dcb8c6b1f5e1d", "dsid_30bd59f3c58a44ce928e4fa154d7aa3f", "dsid_e78e04e8afd442deb3718d2612368a0a", "dsid_50686b8f5c6a49e7989a2b5af4660e03", "dsid_7c29f7b0d0af4695bd20b7720c52e6ea", "dsid_0a2cd37d53ff47d4aced289cd9a76fe8" ] }, { "question_id": "qst_0156::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3c4d006398034c88a735eb9970ea0c2a", "dsid_32c5e7ca0e2e4eadac78b01a35cfed7a", "dsid_d78b80dce604425ebb87797c5835f01d", "dsid_daefe7f0da4e4f728930e1680241e3fd", "dsid_2680a894a35b4117a2220bb308c4dc23", "dsid_b4bd83b46612452e858bc02b380082e2", "dsid_8893403fd93845ab8b0f377916472bc1", "dsid_cefcd42ba2354ff4b5e64b49239d1c29", "dsid_914d75cb27ee4a06b3f69e445d715ab4", "dsid_80a31eab8291416b85c216e5978653ab" ] }, { "question_id": "qst_0096::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8138b96e181940418cc5067b2630064e", "dsid_0f78717a028547e3839243f9c95f376d", "dsid_55d466da21434c6caf6b99680bc330a5", "dsid_8dbd4c431da748629df8c01dccd7215a", "dsid_fd45848bbf9244dbbfbfcc83d1d08c48", "dsid_632f1e5f62244dd99cb6da9069b27a44", "dsid_bde7deaf58cf43c5990a278a6c04d2e9", "dsid_79ccb0c4801a495aae3a4f8004bd9a46", "dsid_bcd39906ef6840d8a88743fc74659024", "dsid_b949140a28774f6181404614e0b722e2" ] }, { "question_id": "qst_0224::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_028f33218175446eb19771ca594ea33d", "dsid_32a4350895a24e799ff59c82d4a37809", "dsid_2fd7a5a549f6438082fb15515bc0e7af", "dsid_34e200fdd069434f96328f53d671dc76", "dsid_bc29223c0d7a4b5dbac05ebf176ab6b9", "dsid_b6ef3b128f7143649196a37d93fec525", "dsid_14733fc073d6426cac3388ee71b8ca68", "dsid_16c561fb74504b32ad2f7afcc4594727", "dsid_88de52e795314a3ab7223f19aba8c920", "dsid_f5f928b83da24545930ddc2818607679" ] }, { "question_id": "qst_0066::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8f5b801cebc34eb4b233bd0ef7c1b546", "dsid_a8d6dbb808cf4c5abe20e1c62bb99358", "dsid_546fa8f856944178a26b64437a128ff3", "dsid_a842f958864a47d79c4e5581edd5d31a", "dsid_5a9b7fae28ff4eec90843beba457a94e", "dsid_ad9b4176efce45ec936ab9308ba61960", "dsid_f13f61d69f5b4691a7289bc37a252461", "dsid_813272211dd4446793c51b6892624a7c", "dsid_b538d114ffc44503bc41c5ccf49f8fd5", "dsid_9fad67220fcc4e6596b8f2cc292872d1" ] }, { "question_id": "qst_0042::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_27d90761ae6a478db1fcd6a29ffd572a", "dsid_96a1f33921374aa6b856dd3eecb95343", "dsid_0271fcc1742547b7a8f587085bd0f54e", "dsid_64d412de139a4e70b9494cf75cb3346b", "dsid_8a27c1f488ec4980a787a5d51be0ebaf", "dsid_6ee1566c1ca1482ba336c97cb4a49f15", "dsid_4affe9870ba54d988c229686a7f819ac", "dsid_45d6cba3f0bc4f9395f85eee93677d77", "dsid_28a0ac3c2505410eb3e0e84468a29f3f", "dsid_c39580d6156a43dfab0c3fdd884f8ab7" ] }, { "question_id": "qst_0290::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b8fb9fd4b41c47aeaa6e009ddc179fa5", "dsid_1b7a5d6230f84eb0a38a97da0e0ce05c", "dsid_dc6d425563f542ba8b91761aaacc23c8", "dsid_acded526a11f4ed8b211b9d5b6d4ad2f", "dsid_fc5e896d8eb4455fa9b186d04e0845f4", "dsid_a3dded417a234c7bb880ac061180da92", "dsid_d47e200a5734412d8126de7b3995390d", "dsid_5f0faf87cc614ed8bcf5fc107af9ecfc", "dsid_3b35497fd1e448d2940dd075c1d68bc3", "dsid_f2c3b068fef8487f874e3c3c90144c8c" ] }, { "question_id": "qst_0042::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_eea41d353de049aa89be98b6f9cf5de1", "dsid_770978f6e8f3414e88d67845f53532aa", "dsid_9fe5b42cc2e045e6909e13192aee7f27", "dsid_786e758a14ca4efbac58f0d18a87384a", "dsid_4807d4c041a6432e878e1a398111feb5", "dsid_fc01b9470b5b42d4a55c230b13384635", "dsid_b4fb062a95c748668ac7e7dd086a895a", "dsid_e08cd8768f4e446fb1c51618c06a5d12", "dsid_359941fac20f446a97b738f7a5f6091d", "dsid_aad7e341fc7344a8bd53dd07dbfc4837" ] }, { "question_id": "qst_0458::miscellaneous", "question_type": "miscellaneous", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9c7f8d6d2f2b4f0d9c2d3f6a1b8e4a21", "dsid_8cb8070d41604b67901e60da07713d92", "dsid_38a4af67314d46179ae4f3e7ff616661", "dsid_bb14e2f3925b4e27829c4f3c1aea3af1", "dsid_5e4346da53e8479f8588feb72040465b", "dsid_6ae2bb84d2cd44bd938bfd39988118fe", "dsid_1e3a1fba0a914113ba6a5e831424888d", "dsid_68d1cb3084b548e886098e653070e99c", "dsid_8a6fb80b25214bfc8d8a7b13179453c7", "dsid_4affe9870ba54d988c229686a7f819ac" ] }, { "question_id": "qst_0242::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_726e843ced0548fe84d197c87ce30d52", "dsid_12d14f86ccc9486e9062616b263cc14f", "dsid_f23926802cf44ae4b85c4410682eab90", "dsid_91481594c1674af6a252e42b34c403ca", "dsid_7a4ca30b507647dcb982ea41ce8dab8e", "dsid_2ab46e75d46c4bc68f3099180a55adb3", "dsid_b7371ab720314b9d8f32b891c06d8983", "dsid_3c18e45fbc064bdeb056d94d621846f2", "dsid_682f6e02cffc4226b3e1b882c1659818", "dsid_dca191e080234287ad86816c46e8b37f" ] }, { "question_id": "qst_0347::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b8691ba2e29542838afae93928ad816a", "dsid_ea2d1b2252a0476fb8351fe49038ec39", "dsid_dab17ee5c3924a9998b17b3591bbae28", "dsid_ddcc9309157b44978465c4a6b8b4c9bc", "dsid_10147c3cc869473f8f1ed3a0ca04a221", "dsid_6626042e2e414e7785cd5e4f7fd96f48", "dsid_ecf32f68ff2049dcbaec9294337da274", "dsid_370aa37572db4327bebd334f1929514c", "dsid_2a6a3fb047a14608b73c57b017b27ffe", "dsid_00318f32b60a44729e010f584ce4ff82" ] }, { "question_id": "qst_0200::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fd1fe7d21e8e4d04bc3f23695ca66563", "dsid_9fd9f39af28b422a99f1f483cca61c6b", "dsid_de3218684e914ce3a0c78fe3c187f9b6", "dsid_86f5fede8ee14dfc886c1e50a5b82fbe", "dsid_2618688a831543dc92b9d0bb51db94da", "dsid_8ee2c1efacb54447a2cef259fcddd068", "dsid_a67e3a5250d04afbb19098dd359f0caa", "dsid_9252e10b88234f368229cf576f1b1cbb", "dsid_47bb2aec2a4b42d28726aa494918f7ce", "dsid_31039756a160403d870af9aefca5ab16" ] }, { "question_id": "qst_0083::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0abbd8a3a39e4cfda70dcc403b22f64c", "dsid_41a817d550b64abdbf9fac78ffe058c4", "dsid_ab09b94479c44887ad5874177ffac633", "dsid_f86330ecd067401c822845816231f124", "dsid_53a865ae4c844158b870c8d083158adc", "dsid_3e97de376e7a4c4d8d1e0b3cb8a80904", "dsid_d6fb522be1f24d2c82c8df2b16d7f974", "dsid_f2dfeaeb52494d11af766b6db753729f", "dsid_9166336e12f248e58c65bf153c046d77", "dsid_2384c6217a874538986694d017a44f8d" ] }, { "question_id": "qst_0141::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_acc33ff059404020b0ff26e19ad2b107", "dsid_0f669a940f374e6ba96cabc93d30c5e6", "dsid_6cd910f09dbd4e8da4b66d17afb9c4e9", "dsid_cf9fb418fa934a44977938808dcc3ca4", "dsid_1fe8dc0642d34c13a35ad33ebccfbb41", "dsid_01fd3e6c481a4fdcbe0b16d0e163b94e", "dsid_b377e448d6b84d598e52e73de0db7ec7", "dsid_e05cf86fbf6e4c0f8393b034b061524d", "dsid_b01c12d7b75240c6a712313d764bd8a7", "dsid_4449456e19fc43918c2db3593c3f1f7b" ] }, { "question_id": "qst_0445::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.375, "retrieved_doc_ids": [ "dsid_cb0d800639e84d68ae035596c7f6f1f8", "dsid_b329231826b04e2dbda9f12b8db13a93", "dsid_c6ec6df4e6d34c68aebead677ba51548", "dsid_2ad75f27c36b492f847d5b9c63492e4b", "dsid_2ce9ae40d5d347c1add2b31c4d5a8c3c", "dsid_723a1f2360884482a857a05356f04ec8", "dsid_16fdeb278bb04dae84e5154d3383c334", "dsid_7b972e9b7ac847bba3f4145f56970301", "dsid_32c5e7ca0e2e4eadac78b01a35cfed7a", "dsid_dbd30b611a8d42deb679ba2f1a69c19a" ] }, { "question_id": "qst_0371::project_related", "question_type": "project_related", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 0.6666666666666666, "retrieved_doc_ids": [ "dsid_7a17252f6044474186155115a7de5ff2", "dsid_fc356e32e9a64c3199be174cb91432eb", "dsid_49c27f8f803541ea9f421d4c573c3d26", "dsid_36d6e7b019084b16a2fa17397e5f00a7", "dsid_a1d8ea3adc68431ebd90cc40a51b5627", "dsid_090a4e39e8cf43eb87341670ea5e3a0f", "dsid_3ea7c0ad13fc498580850ad46ae8ccab", "dsid_25fe5773a2cd4c7ab44cecb2051e8c39", "dsid_2deb330e657b4009a56d4756ea3cb0e5", "dsid_518bff11bad747df93c44247ba6203e3" ] }, { "question_id": "qst_0277::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_7b5a30e323f5400bbd3e00dc4fe9712e", "dsid_c960c001e9f2479e9ff9567a2c1d795f", "dsid_683dc2cf9bcb41b7be76949c6e0965e6", "dsid_ffa04a7d7c9b4c0b8f962d23d8c08c30", "dsid_fd9fd8ad97c84342bcda01f6bd68cced", "dsid_c3d1d105d0524c509218a2103848a6b0", "dsid_04a97bccae6345f08443c01919081b42", "dsid_0b102079bee34d4bb06eff914835aba4", "dsid_3d9c92fc4a154f969a1754c78a53dd97", "dsid_cd6e4153efe34c19a4ceef954cba0fd4" ] }, { "question_id": "qst_0322::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_bab6aba4c06e41e9ae46b9355f644a7a", "dsid_dd34a1f6919e456ead57b0c3adbf4d18", "dsid_89cf9a5275164e75bb891679151e292e", "dsid_e2141c0a04894fc3b47b33f8e06adf5d", "dsid_e4427606e1eb4a0d8192b8bff51d849d", "dsid_d20e25a1a30b43f1beabb99439944366", "dsid_14106824ac654eb3b84f5fea7d9fdf62", "dsid_2619d808a84f469e96acb6934752a177", "dsid_6258d935f1714b339654002609a504cc", "dsid_3245ae1fc27346db802f09b997478188" ] }, { "question_id": "qst_0109::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c9e70cb6eee1471baba213a976a8dd05", "dsid_2de9c2082e074140bef60a671fdc501c", "dsid_14f73462acc24ff09bb8bbfefb4c480e", "dsid_3d4654db2ab947fcbb877b8d0afe6925", "dsid_828c7f2710b0435c85cd004126aa209d", "dsid_92289c54d6b046c0b8495394f4951acb", "dsid_466d946096234b50a646d0c896e9ca00", "dsid_176b5cd2c79e48dab2b62123295cc364", "dsid_1d60c216fa7547608e273c4caf841312", "dsid_4ebaf545e97d4c8981b2148c81da0a5e" ] }, { "question_id": "qst_0294::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2973d8c94a694a2da87673c284cb7c04", "dsid_88a4bc9c35ba46dbae03e660a8cebeba", "dsid_af9f593080734421b56db8dd98d9ed6e", "dsid_741e17f34f654ee7b1fdea7eef31af8e", "dsid_2e27ba3769084156b53d050a46e93fbf", "dsid_ea340c97ba4f48d0aef6e76a7b11bc18", "dsid_defe5edccd69488e8cecdf2876d3d0ae", "dsid_1eb5ba51d1314bfca8b586dde03205d8", "dsid_c138910a3a0c48a0bd1bd3d7d9edd896", "dsid_4ad360e4e9a946b69eb10b4ef22c7154" ] }, { "question_id": "qst_0284::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_4eca941be62c484680660935b66848f9", "dsid_1d7e3986d1444fc0860bc7e77d166183", "dsid_beb98beec7344a2a805997e1b58b1351", "dsid_eef545ab19674255ab816bc1721d2026", "dsid_edfb9e1f31394c1ab7bd60c1fcf2679a", "dsid_320b9c2050b44a9194ebc681a9cd24e1", "dsid_a97ee7d15c8e4cd68ffcb633c599dcc2", "dsid_ca55c7c0fbfb43428bc19aed35188a55", "dsid_aad99adbf6af48c38ba182e212293f14", "dsid_14bc895cb3a64c71a3632a743dff3cee" ] }, { "question_id": "qst_0051::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b7aa9260ff1b4eb586edf78ba2ed9ca9", "dsid_72b2786c94ef4143a947759ad7a071f4", "dsid_578a4cddc1b2418a8b3011f6563dc214", "dsid_b7e285fa5700488fb87461b56db5bec7", "dsid_0f991287334d40fbb4d597b305447fd2", "dsid_e20d9bf6a75044189af72a9b0062a478", "dsid_59f0e609869142e79a0fc487c14f3819", "dsid_b61151b44ec947d0b23cd6776d29ae45", "dsid_27b00f8153e249f4b917f21efdc49024", "dsid_e8c33c9af2f0406f874871fb2e23a62e" ] }, { "question_id": "qst_0016::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3f7f32874e9b4ba49ff4fe28c0ef5de0", "dsid_3848df9a653f4b2da022ecb753587654", "dsid_18054493573f43aabcda10beb6d2be59", "dsid_f125690b111647cd899471b2257b51cc", "dsid_a2c21138d0a346f3829f79b8294fa762", "dsid_1b17615c99b74b68b50924e7bdd8bd26", "dsid_4289f3f1fb54444f81ad6fb1edc6dec3", "dsid_d6236d3b3d314545a39a6cb8503e8a2e", "dsid_5def276e94ea4202b7f15360e3dae44c", "dsid_b04d99601ddd4b528ab88ff5be1360f5" ] }, { "question_id": "qst_0259::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fc009de77554459b89e6b4340e8eca82", "dsid_2dedc3d8d3684c30af553326438a0ff5", "dsid_6c09c40e9f404f9d89dc4251145307d1", "dsid_918a2ba9aaf447ea9d9e076fc7212af7", "dsid_6ce84394f2c74b0f8a5cce70d204d852", "dsid_67cc82eeb08b4b65a05d93e1919a144f", "dsid_bd2a2e090a604950a5ed6226f82400e8", "dsid_28bfb0834073495faae5d53bb4daea2b", "dsid_662978747ddb47338b62f7d87c391ca2", "dsid_8aa30532ab3f4ed5adfb7e9ef0c9e808" ] }, { "question_id": "qst_0203::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_83647b9fff63443896977370fbd384ac", "dsid_8a175e69fdf7472384c657448acc9472", "dsid_3407ac164f6744d4a2c241e0c5412b59", "dsid_c35ce749b81f4bd59537a72c0e26f587", "dsid_e69a1b81fbf94e0ea27797f2232530a7", "dsid_d34fce9ac3104a509da5248500ae03f1", "dsid_800c4280d8d74533ae12d75c77b1efb4", "dsid_b637cca914da4a50aedfbef2fed4bd86", "dsid_f6d58fc9be6e4a808ffd6822333333dc", "dsid_05559b70f35d4898b9b25fc8fcb974ff" ] }, { "question_id": "qst_0408::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_e190b0de0d27466fa91bfc18b0815311", "dsid_5f119369627543848ddc03d6805509dd", "dsid_e4142c27bb5447bc92a280a8bf1c36f9", "dsid_54a976f38c8940efbeaf82dba5b6ecd9", "dsid_378ac277dfa744c8acd16b30af9e364e", "dsid_190d9732b59b411d8a6bcc59fd61b309", "dsid_5147667611a54584aa272d1469d8f6a7", "dsid_5e42acb61ecc4c5599ffccd619b87ea2", "dsid_fe7a44f8f7f943f987513d5a756fc013", "dsid_6b45298647c14466a16d0b559831b2a5" ] }, { "question_id": "qst_0126::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_dee213d420dc421dbe3cdfd9b96f4a37", "dsid_4835bb2555f54f62a2370e1c179fe95c", "dsid_310b703bb5b04dc7ba499dfab083c8e2", "dsid_22d7ec6499bf4eeda5ee8d1c3991be04", "dsid_be7635d0eb2247d3aeb05734ecaf5b42", "dsid_2d82aa9cf7d34bf8a74d6dac01a17a8b", "dsid_2963268c0a4b4ccbbeaa3492c054a9fc", "dsid_cf3cc96da74d41b9b9f2a54192750ba3", "dsid_4f6e89b6ae494b8bafd5124c82950a70", "dsid_ec1b28cb66f4415d9c774ed43fc54fe6" ] }, { "question_id": "qst_0271::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_db87c0dec8444b1d83b06cbbe725d3a8", "dsid_e405cd8a5f2447849fe200504b5a39d5", "dsid_645cf10941d94bbda0d9fc6ac673e883", "dsid_5a185ba825aa4a68af428f5ad1aec7fa", "dsid_5da25063af26461994ebc6c0013f9432", "dsid_638bdb9278184f4f873b02c6e55af122", "dsid_9851dcd734bf45ccab6acea0b9b51f76", "dsid_c030f56acd5a4447a79be25683afdd87", "dsid_a5b3feced5f34074b0b29d647519e3ff", "dsid_aec40ea3f74b4984a48d7def0a094608" ] }, { "question_id": "qst_0135::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_63a1c6c12e1e4d9cae67056673633192", "dsid_a959b5fcfa14486282145b8cde138f2c", "dsid_a3f4a0fad8d64c19bb1abb4bda52395c", "dsid_ad3961a5bbd846d09d101bfa951ab739", "dsid_6c27b75e4af44d5fbcd22810ee6a3392", "dsid_69921bbd7dc4415197062526429b76ae", "dsid_bbaf064b5fbf45e4a639c2e51830fe5c", "dsid_7a748fda90224c479d7535332188c681", "dsid_7b97733462e14937af7a5abf89577593", "dsid_4a6d655db9e3439da58cb2060dbda494" ] }, { "question_id": "qst_0065::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5e4fced5bea4475c83a105ac26aeb54b", "dsid_adba0fff3cf94faba1a64677d66aa19d", "dsid_1eba87b0b11b4f31a006f0cf164b8509", "dsid_4b2c6c600488467ea8eaea64fd4e4aaf", "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_964f838c97bc44feb56077e3d67ccc04", "dsid_8dfe3e03d22d4e61b79b11b6012cc249", "dsid_46b0bc1e97ae4d8792d08e6f4122b948", "dsid_46383d57889e429380b4ea4d9bfb5646", "dsid_70124860f38d4604981c3c41c45308ce" ] }, { "question_id": "qst_0050::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2bea8116212844eca26c9c90052b0ec2", "dsid_089917448e654911bd186650ed12feb9", "dsid_f1a3e4d11e274a22b0d5e646ad0ba9f8", "dsid_cd7be68e755f478ab32333eb90c8e5a4", "dsid_aaaf6ae3258d4fe2a6883f2cc58e7360", "dsid_09b15c8027114823b855e6d135df83f3", "dsid_be8d2802cf3245088a7e7f861f317b31", "dsid_11c048aa9480427384e13ab303e85512", "dsid_9626f1c17c854c3291f02e3b1d3db535", "dsid_d8f3f9dc66bc47d3b087f6a772582e0e" ] }, { "question_id": "qst_0157::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ccaf626687b940f3a7309e09c1ed5c1b", "dsid_a9152bd748f342999bea434d517122a6", "dsid_2132c5730307400bb971dc895c00d1e4", "dsid_0753a5dac4304fd4813d124865b80174", "dsid_3650c6099851415d8711e83c8cb6ad33", "dsid_189913bc1cc94a799a412fc024044c5f", "dsid_4c3ffe5a1d9b44bf8ea8460e42f5b286", "dsid_23b264ebc5ee47979bfce01157b2b2f5", "dsid_e15750196c7643729e122d2e1cd7fdd0", "dsid_e998a0c2e7ec4e78a650b10dbebffb8b" ] }, { "question_id": "qst_0240::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_4f3e25d63e044e3091fc4ab0d9c5a138", "dsid_fc34f401b1724cac87b02b53c49b14e9", "dsid_5402831508734a51adf649603be4b075", "dsid_94098e0c64e14361866d21da3bbbd283", "dsid_9ec31084a81044eea561dd829b8b117f", "dsid_b784e8f2138b47c5ba14a93cce75b68b", "dsid_c2e684d1d34340e5803e4d08d2be1e0c", "dsid_ab216d7d343d4408b7735bf5081f68d8", "dsid_98cc8d5d25c340218b9de9a71f5ca629", "dsid_a9b74f48b38d416b8a71bd525726d7eb" ] }, { "question_id": "qst_0094::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_82e08a01e79b4b8aabd6a957097a0c56", "dsid_e4da9a6e464448cb94972d6259d476d8", "dsid_bec90d2205534ca9a016f244e13884e5", "dsid_b793195a94004991b5ce288b087c7b8c", "dsid_41666cea1d644dd7a401dd6d416c9fee", "dsid_a4c42dbf64a540b4bcac55013098b6cc", "dsid_1f8b9d86e07a4f639d487a74a7971ed5", "dsid_5016f308bc4f42f3b9f85c90e40c5b9c", "dsid_a4fcd10fdd18477c8a3ae4ec2f34c68a", "dsid_98cc8d5d25c340218b9de9a71f5ca629" ] }, { "question_id": "qst_0080::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ad3961a5bbd846d09d101bfa951ab739", "dsid_deae60bb40f445ec8b0416aaf0269253", "dsid_68d203c17f6d446095947b4788e8a083", "dsid_a3f4a0fad8d64c19bb1abb4bda52395c", "dsid_6a118a3fa8034c78b9c42303cc5440d8", "dsid_f619f7c8041e477aa46f6d7aca953528", "dsid_dcc7045dcc964ac8a58bb29b0270e2e9", "dsid_b135486896b5442bab43e66e6a4191c4", "dsid_682b716dd06649cfa62d2b5792650cc9", "dsid_4a6d655db9e3439da58cb2060dbda494" ] }, { "question_id": "qst_0404::constrained", "question_type": "constrained", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_5b2aa3e1e8b54705875b9996322d762e", "dsid_b79c7fbba1884ff487bd2a4bb79b5598", "dsid_025f5bbe435748ae9c67ba8a45cf608c", "dsid_cd0a915e779b494d9a1327269746d51f", "dsid_cfc5bc49b0d4481f986d0c058b790706", "dsid_e13f3074d35f44b7a3aaa697ec5fe125", "dsid_b9a11bc85afd4882b64b78cce633513e", "dsid_cca9b1e521df48f1834a3b4af85ef161", "dsid_9317f78d9c364ed88169d57e23271322", "dsid_adf6c58b4c984788b5c4de7721e33346" ] }, { "question_id": "qst_0164::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5c05d2a74c484906852925d1cf1daa70", "dsid_5650ae3bff08466f92289cbc565c7d2b", "dsid_0bfbf30b70854702be9eb768f2fd7a62", "dsid_0a2cd37d53ff47d4aced289cd9a76fe8", "dsid_60b6b4cff8f440899bb213cbfb4e90ed", "dsid_8d97d64cfcb14cd3b13589b86b0a0c02", "dsid_88ac13a2119b400d9778cf37699d8ffd", "dsid_ec6e3137b9bb42eba7271e99d5c8d47a", "dsid_7353ace7df9d4dd492097ab15cdd439f", "dsid_6e81df5322e4483abbc2577c9ae93e96" ] }, { "question_id": "qst_0312::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_0646657fd7e3448faf959a6883257e23", "dsid_c2a137fd3f114c268cad5d2c323af502", "dsid_a449c9fb7927461d8deba42ee39d1921", "dsid_95aaab5e5a6640ffaceadee4bcdb5f76", "dsid_635fd169c808444987f7de6af193e8eb", "dsid_13e85f84f35d45e8aa472cb085c7c1bf", "dsid_fec25e7f54c449f69149a71abe09bd44", "dsid_fadab57fce044302af798a476431d3b7", "dsid_a46fa899a7b2409eb29b34848061e9b5", "dsid_f6440d3564ea49cc821ccf2ca3dc5036" ] }, { "question_id": "qst_0215::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a56c6ba5a2094d8b83c6c70d025d940e", "dsid_e098932bcbac4db5bab53e5eba1c3dbe", "dsid_aa9506249505475999f9175cef0abbf7", "dsid_d7b6b915ba6a47d98fab7dddcc43e50c", "dsid_2463eb6e5daa46f7bc51be33788108d8", "dsid_3a2cd5b761ee48639842916ab3dad744", "dsid_8ed07df6c3b7411d94410c1584f67a7b", "dsid_f135a85feaf3453c93a0d78ebec1d457", "dsid_a16bcfd8386143f791046fa52b672669", "dsid_68fbf6106e38493e94a832e218fa14c1" ] }, { "question_id": "qst_0265::semantic", "question_type": "semantic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3e9d1abb3e9041c297f6e392d7252265", "dsid_033bacd65ffc48c59ec27881b70af6a2", "dsid_4866ef84859a43b89d77178c28d9513c", "dsid_876b1a31bcc7409ab560b9ccbe5a0d41", "dsid_bad6bd97d4db46a59cb15721b6537cd8", "dsid_4ecb42b192a1421885f6fbe8a7cbde8b", "dsid_b127efa7f884490fb4704c90cc8f0592", "dsid_986f0466ed5640a09902c5e71f091ded", "dsid_4e3018cb7ac3465f996af7d1d2c6d070", "dsid_697a5537f53d4a46bb27633ce1b84683" ] }, { "question_id": "qst_0139::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_68d4086cb31a451a9b814f1d0c467c79", "dsid_1d2f3315027d4fc9986c327586b5d0f0", "dsid_5d131e8a1e174c21a38fca105cb05402", "dsid_be3e3fd3fc85489282359680e8004a31", "dsid_3b5f7f9794cf4f87baa5e1b1d582ded9", "dsid_6fa2d4bd65574007aa28054bfef81f9f", "dsid_ef3734f3d94f4d0e805af2da78ade930", "dsid_b6ba77d72e0e41fe82563675ee5a7a96", "dsid_041ba5ec74e844acaa76df4af273399e", "dsid_17a62c9cb9cc4479af039f23e1cd7c40" ] }, { "question_id": "qst_0369::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.3333333333333333, "retrieved_doc_ids": [ "dsid_e7d8c585e8594fe481975515246d04f2", "dsid_31be9c3c708a4a5797d50a579c137587", "dsid_a560a94faf9c40ec9782d125d94fc009", "dsid_5f67d44565c54a808ea851c9a049c2a8", "dsid_a360682237464cd196d9c439374d6520", "dsid_30a566647d334042987532c91410fc97", "dsid_38517717fc6848e8a3a4302b303af302", "dsid_9a7191d7761642c8b2553184b5664677", "dsid_6e528fa402c34043897649be6aa6df3d", "dsid_1e7f42e5c50e49a69f8ec1a1d8e2dec2" ] }, { "question_id": "qst_0403::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_a98ada14af2142a7a46266584d05c846", "dsid_c1b16f241ebf46caafdd845edd2c3da9", "dsid_2b547cc853804444a811242d53817f74", "dsid_242661c8aa1341148e493604f4733d4c", "dsid_b11d65828e2c4f379452c7698c8c28d5", "dsid_8cd8888f9a1e4f9d9799b0331a59e620", "dsid_c9c526d393f34b27b4a649eba20301aa", "dsid_3888c4df0ed9442fb777203855558f6a", "dsid_4d9c384698264f76a703fffce5021cf0", "dsid_c8a4367928d447d0bc86535046bd2a04" ] }, { "question_id": "qst_0194::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6049efc70f294b48bff3cbf52ffeb128", "dsid_0c8ea3b2a7fd4392b1f480fe5f0c4725", "dsid_86d0a27f0f5e4a4c997e140b73f81150", "dsid_805483a9523f4f499d5cbdf72edeabbd", "dsid_99d1fbf95f9d45e699c4e4327319e6fc", "dsid_f03bf31eef8a4785b9e65ded4e13a89a", "dsid_e23bd5cdf6e54621a9ffc1766c41b51b", "dsid_8d33d34055924b56851c12534ddc67d3", "dsid_d41dc7d257a74f9fa973b17d408ae91b", "dsid_940f70c0190f46e0b1105e74d3886ffd" ] }, { "question_id": "qst_0214::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ecec45fdb060420ca809f6804e8e3b78", "dsid_328b258e0f664bb5aa545595343dc2e0", "dsid_303196c831af4d1bb38a1e2546228a09", "dsid_397ed85c900449dcaae969f6823a395a", "dsid_14ab5ed61abf46faad167f1c56e16abc", "dsid_1d9d6740f21a4943a4391458aaa94574", "dsid_645ddf2de07840059398da159b5dffc4", "dsid_b98d76cd1b1242b6ad666c6fcaa98846", "dsid_da410d4361b14844a3674a3426b61831", "dsid_23dac50a480947cf8015674217eb9c04" ] }, { "question_id": "qst_0018::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1f81f208f51a4aacaca226109bbedc72", "dsid_61dca8c9612c408bb65c5c7d446d96fe", "dsid_227fa7c5a0e8490aad48847d735bd196", "dsid_568e757465f44d1db10bf120f977a0c6", "dsid_9e164d0d18854df2bf884cf485d4f05b", "dsid_94d3e0e0930643a0938d52e67b52c8b8", "dsid_1c8defaa24cf4552a28875d8bf101274", "dsid_20f5bf595f19483d9cf96517cd345c9a", "dsid_3ea4b2fd67b84a50a67232c8e3901431", "dsid_09f0c8ab37c44ad18a24d080835f25c7" ] }, { "question_id": "qst_0175::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f02a338c2dd546b3b025b5d5d429ad4e", "dsid_6d555ff468544dc3972e3fc680748cd5", "dsid_a5f01bd635aa4596aa459e5f0bf208b5", "dsid_b2b58499bd844ec3a41e287049559716", "dsid_f832c706b1d14b97b514f327f6932664", "dsid_3ff8acce071d460ebecfa1b2faaf7b12", "dsid_1ae853294fe344b7bd2d776176091b88", "dsid_3c3f004beb0c4325b1321dd5512b464a", "dsid_f5c4da3ebb8146f194a5124d392e2624", "dsid_8c5e036efa3c48ac94d84d7799f0ab92" ] }, { "question_id": "qst_0160::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c84c8581594e47d49e82d024a6da915d", "dsid_409f56cd415b48b09fd0187ecdb48b1f", "dsid_4beb4aaeccdc4ddd989d4d6d23f4eba1", "dsid_fdc3fb9502544d62b8c1e782d0e73bb3", "dsid_d17f040352fc4b78bde0e93c6c4a3e4d", "dsid_80c521b2658f4e969d712c472ef65383", "dsid_3c44e22f5e814eb38fbf3c1cfe29bdb3", "dsid_251448c5a37f4502996a3ad218f6bfa1", "dsid_eb6986a727f64742b70b58e928150593", "dsid_ef04f4c09a1b498dbfefd2ffc503672c" ] }, { "question_id": "qst_0063::metadata", "question_type": "metadata", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e555234794034e218991ca49a2ffb3ea", "dsid_c1f99c50101c486bba6926609637b94d", "dsid_c2b48ec4a11d465b94744215b661b6eb", "dsid_c880448fba214acba0e50dddccc3242e", "dsid_6bc0b574f82b4e8c98b1cd5fa4dbe9a5", "dsid_62c87e92c7a44e269d652c3d8e9bbc91", "dsid_4814eda6c5574240b1ae4a8f3e01dcd1", "dsid_15dd15bfbd0f42c485a9b5b9ace6a3d1", "dsid_b8449ad4438c40b59de29b4511c6bb36", "dsid_919f817729c245adbded80e87df42f79" ] }, { "question_id": "qst_0062::basic", "question_type": "basic", "rank": 5, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.2, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b193c79da1894b548046571c89655fd0", "dsid_438eb5db87234854a830d0cdb638153b", "dsid_6edfb57d2f384f1eb2a0004d442b7800", "dsid_85203717c8834da6b946b329f3258f33", "dsid_702bc520c52040428799fed41c63c8ca", "dsid_f1a3a25e32954230abfc12672309cb32", "dsid_bb4965020efe4feb9364a4dcd5465f66", "dsid_e532962507a14cee8a85abe952e8e788", "dsid_fdc47255b9ab4d5fa11ab6d16e46be37", "dsid_1578aa4ec56d44cda84da21b66ab5a02" ] }, { "question_id": "qst_0096::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ca09c9594d0f4293b971c96fa9ed647c", "dsid_02dfda7a1b864bd0ac476e04815ecc94", "dsid_5c1c408fa554409fac6125db3ce9b387", "dsid_86783f1b3d154ac7b57c6d3d71bb5efc", "dsid_380aac4dd4254d44a2b72d253fe0e268", "dsid_8d2187b4c5e144b4bdb07f13bea1eacf", "dsid_82d50eb5c3984502bb8b4a50fb9b4dc9", "dsid_236c4819761f47bf9313ba52284632f7", "dsid_fb164b83b4fb4a4d95c90b0831b2a53e", "dsid_4c3b0b28fdb04170860fa7130f328892" ] }, { "question_id": "qst_0082::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_dbeb3950ed074b20b8112e60e9ada150", "dsid_43b87c65f81b4e2ca03ef919c456effd", "dsid_9fad67220fcc4e6596b8f2cc292872d1", "dsid_b558dd44aa6141228f8cafc23dcbfe89", "dsid_b1287d634ce84d20a643e2d20af168a9", "dsid_a30530b978414aa6a5c2a57fbba80c6e", "dsid_02d2a67ee73441b3ba296acca994773e", "dsid_deb8e9b97a8c44e5bb071abbe5053311", "dsid_b8c5ba3b4f56485b9814f459f52679ad", "dsid_6d1ea984d38344739a4361169b099975" ] }, { "question_id": "qst_0010::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_97846a3d292e4c778a0315c1b398b97a", "dsid_e16c1632730d46a289494796000c1a24", "dsid_dc1807ac218043ddae77bbcc8a7356eb", "dsid_c5806a46769641af8474babb5e19f629", "dsid_04dca3e2f4164b589ad5be0b53a79689", "dsid_46304695ee8144c7b48e2393510ff03a", "dsid_515cfde3929d42efafcffe5c79c65b56", "dsid_be286e63a11445edaca7630db846442d", "dsid_ae77d33628b248f3ab828bceded69001", "dsid_7be67a8a347b4576be26a5176797bbf3" ] }, { "question_id": "qst_0391::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_594a81b3332945e7bf596875af810ab7", "dsid_ceae9e544dda4d3b8257842be18b59b4", "dsid_6914aabfedf04fc582d821388dcfbab9", "dsid_6b68cc690aed407bb180d40e8640a453", "dsid_9a7191d7761642c8b2553184b5664677", "dsid_775466ac57a64f9db12b274a52790aa1", "dsid_012a01961ac74fd28a829c2bb7850573", "dsid_f732f6ea48a74d74b95a0df71c48cc6d", "dsid_79f69bac235144aa81ad4a45b66f1410" ] }, { "question_id": "qst_0288::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c4db696d170b4c38b80baa91a45bd4ae", "dsid_a4215240075647cf9511eeb70fc2564b", "dsid_c2ddd5240eb04e16806d82103f7646dd", "dsid_e78e04e8afd442deb3718d2612368a0a", "dsid_64a76c731a704b7390dce7f4121fd93f", "dsid_7205e01bf69240a48451b947589dd52c", "dsid_7c29f7b0d0af4695bd20b7720c52e6ea", "dsid_c790a6aab4d8476aa62fea59d6e47cbd", "dsid_4300d29b15364fc782a5d279a3e965ee", "dsid_5312a36dc1a448a8a09bf2b0e7e7967f" ] }, { "question_id": "qst_0138::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d873f1adbe654259a48b9ba497e7e4ef", "dsid_ea1a168728a94056b4897e5a9dcf133c", "dsid_cb4d4827c76848e696fd896a18cefbe7", "dsid_4ccec796173f458889b2801084f777e7", "dsid_6f86b12eff0d4a94bd6118e2439057b9", "dsid_7e8a320d71e34cca9e24059fea1d0cb4", "dsid_3237666f5dd74914b1bb5cc5dd7630b2", "dsid_93e3eb0a2ddc4892ada5277064237654", "dsid_bb4359d8f433401ea4fd6c4743f0f1e2", "dsid_35144b08882d4815860be37ca045c1be" ] }, { "question_id": "qst_0420::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b8aaecf54566449095c32eb057d0d267", "dsid_5176e8cbe14d49909e00764bbc11b6c1", "dsid_c9a936bd02d740c3a528d4373708256e", "dsid_939397ccfcb94c2bb22c31658c447b31", "dsid_5049f6dbeecd403fb766ca8305d56ad7", "dsid_2b6f25233eff498688a91b0be28c8d09", "dsid_a3303a13cb834f8bb6092e3e3febd30e", "dsid_25d4b562d1a34bb99df6fcbe8a7574dc", "dsid_f750c0d48cea4255acbe7d260c91d2cb", "dsid_75ae758fddb8494eb81f572b3f107ddf" ] }, { "question_id": "qst_0462::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_df79433aa56c489d93efd96e27dc1fc8", "dsid_e62a534321a24da2a93a003a9546804b", "dsid_b6fa323be1184c63a0536502438b6bdf", "dsid_cde4dd69fde3485386f8719b73bdd2be", "dsid_3d2ad970b6fb416eb019c4bc6c3c9370", "dsid_e41e27e6f3074276a0dae938018f8998", "dsid_cbe7722f2e1f44a0b96eb044213006f3", "dsid_2b9bd971016440f2b0415dea55dca1bc", "dsid_b72133b826ef4dbf833b558aa9e7b4f6", "dsid_da7e13b65f8a42b4975b44b2db0a36e4" ] }, { "question_id": "qst_0236::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a69d13b41aa2432fb0f59f20e22a773b", "dsid_a9bb70a65894469ca2cc97a8f9dfd7e6", "dsid_6a1da8c34c1142058ab69c3537bff385", "dsid_e7de5a80bab945c5b943d8da0de38ae7", "dsid_74e2046791184ef18f5775ddaf563383", "dsid_cab05f63321746d69df0a1e79ce31cd3", "dsid_2f4b42ce326a4b39a3558f97a9f8fedb", "dsid_08dfca1e608a413fa3b4c9bcbefe3f3a", "dsid_1247cac07dfc43efae8025f679bd37b1", "dsid_9e578802328541b0947b8f64fc4c6cb3" ] }, { "question_id": "qst_0464::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_199789e68ef741bca44a24d535ed6c00", "dsid_9629e3c800de4029acbb92d2922c2e52", "dsid_c1284a3217fe4b2f8c874471705cc47a", "dsid_c1f6b8ef13394cc8a1e8c6fb5d470c13", "dsid_1124747125dc469c947cfaec10a4b6bf", "dsid_7ab42408135d4deeadb30691c20a1606", "dsid_307157bfab1740e48fe183a1088c7180", "dsid_f0b58047fcf34a94a26027d9e77b4dd4", "dsid_58aeb528547a4f0dbad310e4a0d0f277", "dsid_ccf5fe7c83a44057912160d478cd9c79" ] }, { "question_id": "qst_0373::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_16996cd96c1c4fa1ab399cbc446b186f", "dsid_9b30f5e9d84346fda6017553e8a78932", "dsid_5324918997644ea090703e088b166214", "dsid_0771b119d60346c2903084d9069b6fe3", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_a8bb0a59e5bd4279969a8378675672ac", "dsid_989b00c7aee2497285a43c4045e5217a", "dsid_66ef6e93d06b4f28a69746b5eecac1c8", "dsid_c717ac269c124c5b95b02889ddc8b3d8", "dsid_901efc57c7184a3ab4879539082c7740" ] }, { "question_id": "qst_0382::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7d347905549f4456b0d2226b47e4aa68", "dsid_94f03d3be70e4590898fb4f6e20b5d44", "dsid_d3584534291940eda0232157949b0f16", "dsid_d04dea2d736445c3af6934b900a12ef9", "dsid_c1a88137436042189c6fb011ece2b760", "dsid_445fda755c8d404f958a22dab1fa2d2a", "dsid_c4fd6edf855a4dfb83fd01d398418324", "dsid_7f69a4a3c0244fd8a26fb6e7ed391d53", "dsid_423667a90e104c13a90d6050e4cf934e", "dsid_3c6da06cc109465fb66c7998023465a1" ] }, { "question_id": "qst_0040::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_43ecb807903f46848e8d099b6c8beb04", "dsid_423906d961ea4729bfc4a5e100476587", "dsid_6b5e244611784255a6a7328dd45a4516", "dsid_e51021765c694a65bd60b3d38b12db6e", "dsid_295e47cb4b6f43678091239ef09faf93", "dsid_484737333f0b43b5a1290c7af5beb058", "dsid_e48bb216a09940d1bdc8459b54299413", "dsid_8872d534e4e144a4bfdbb247f9f5843c", "dsid_e734542973ce4273bf08f569227e6d47", "dsid_a72fc5197c274cf283b3e18e40c4a275" ] }, { "question_id": "qst_0323::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a2c7dc70aacc470894c953b6c7499228", "dsid_22d4b181f7ff4a06a983263de150bfb2", "dsid_5a676c302f9443b9b1238f6b627a73d6", "dsid_dc2cbd6252d54a078187c00da64216d0", "dsid_78a2276db4be452f92012c84c79ada9a", "dsid_41f825d6398e4a6c930f2d1f6de7b86b", "dsid_0feb64a2a0b242458a41ed291b393cdc", "dsid_b6a1828946254f6b99c84cff740a9d11", "dsid_83cb5de371004200b9e2cf1c826087fa", "dsid_6fc91c2c69a046ab844fc0e9de246e1b" ] }, { "question_id": "qst_0003::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c5288dd4874345adb80f4a71c9a18773", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_b06feeb73e3e4175a981a645af353aaf", "dsid_c0a3a63c0d2a4eb6bbb2a30db7c571d3", "dsid_b3b35c6e716945d39940e92541c8bac5", "dsid_9db5a9acfd274043beb71a6308df8546", "dsid_e81867b66f2341929bf3df0ae3fd5884", "dsid_9289f141b64245bbb549bf8f27b274cc", "dsid_9af635f923b946a08fea28b6c6827f55", "dsid_2b2d761eeb32477599aab4faf48c6c1b" ] }, { "question_id": "qst_0150::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_029e9c3544a54c0884054d84bcbcd9c9", "dsid_a75a4d9aa7d940009b7966188753d520", "dsid_6b4a5e371656429d9005e289b36bac74", "dsid_529876f3c1a240718ac80b9c05973e21", "dsid_625ee72e61374d6fbbfb63cbcc2c508c", "dsid_a8af533a3f89402ca739bfdc5322b42e", "dsid_4ec9afce883340a88e49575ea59dcf7f", "dsid_f940e7c95688406cb497b0b1dab44e99", "dsid_da28108badc24c84bd2eb13aad8bd9d8", "dsid_1df914fc6ee24bbd8e072e53490047a9" ] }, { "question_id": "qst_0134::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_cdf9b20094574e8eb84dfc5f8a1169d6", "dsid_c42820ebbc9f4ad8ad04f2c9d1fa4faf", "dsid_207f2f98c14e469c8bbc88be8f328f0a", "dsid_b49e7b9a626a4f57a69d97ef12a00ee9", "dsid_138a982ca16140a89152a5109f218dd4", "dsid_4f0b59f1c83640459247deabe08dbf23", "dsid_752c90832cd94113b3c5297e74af32b8", "dsid_2f65523ebdaf4e3a99484bf321bafa88", "dsid_a247c25cea1b42c08456004eb33f2faa", "dsid_463f3addfb2b49e6983b85b49a5865cf" ] }, { "question_id": "qst_0453::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a8a5eac6e7a243319426769963506277", "dsid_a254e962c4b64a159b47faa3c8758592", "dsid_5e3b7bb4753f4419ae8814599c9c5c53", "dsid_25bf55b33a9e4d29915b4c61dcf5eb49", "dsid_90ac11104294466b9772b17ee638d4f3", "dsid_c6cf7bb994ee47f0ae87dba82df67c24", "dsid_5356ee72a58b4dbf8369b733904b880f", "dsid_99ee2f1f290c4e6881ca562c61837d2e", "dsid_bee0aedaea6f42d6aae86854a6057d38", "dsid_89653764c0ab4fa5822dff69a33258f7" ] }, { "question_id": "qst_0423::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d417b614467040e0b9f40ab37dece165", "dsid_dea9de3e5e904e4aa4e173e254e785c7", "dsid_4a2476f041294cd6a9fc0344db20a553", "dsid_caaa763133944643b120dd51469feab2", "dsid_c5f0f39c0d2e430680b1b12b523493a4", "dsid_b0e1344ddb2741d8a985032aab472179", "dsid_c6d040defe334c2d838f9db176bfb7c5", "dsid_ed47da783e764157828c030e9778e69d", "dsid_6c1cf94ad59742a499293dea5dd7fb28", "dsid_f59a0860f18c49acab904cea19b3a610" ] }, { "question_id": "qst_0038::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d04ffbe63ea44224892533eb0fe95bc6", "dsid_06e5984cb6004090a898923bb50635ba", "dsid_685dd9516ce84945ab84837cba7b980e", "dsid_8370a3e01ed54472bff4e036ab88a53d", "dsid_3ecc12bb1262430697c998b43bda1bef", "dsid_4f53cec258db4f4bb6a9594c752c9c38", "dsid_d29fc29f3b7941e880b5004f32a4d07f", "dsid_2989d423a9384e49a6df052170272202", "dsid_aa5d7005a9fc4703be067c012d4a81c6", "dsid_cc81938e875f482da37226102a8913a4" ] }, { "question_id": "qst_0392::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7fd136dd100c4c9cbbe1958c94c26152", "dsid_03fef9fd76bc47989e476fbe7f11a36b", "dsid_5bfed526d8144b0d8adfdd0bb5dd5419", "dsid_ebdc776e4fcb481fbae68e9d7d5c767b", "dsid_5aa1223a107c4d7bb3c331a213811678", "dsid_dc5f484a1a844bc2aa450f949a449ed8", "dsid_258679d566634d4092bd5393f349ad04", "dsid_f8e543505c1b441aa5ff55ef4c7a1618", "dsid_65bc45fa1991455a9dde2c1aeff083f6", "dsid_265b811cd2834a31a5be5bd20d425754" ] }, { "question_id": "qst_0091::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_59f0e609869142e79a0fc487c14f3819", "dsid_65be1880df20486fb86448f27287ed90", "dsid_6b3bf537e0b54d76948f36c106333492", "dsid_2016c4fb92394c21af9f481e0fb4b3ee", "dsid_1f6462e38c094fe1814f9bc981925e38", "dsid_f8c8a11bcb224dc0a68dcabb8aeb2a38", "dsid_578a4cddc1b2418a8b3011f6563dc214", "dsid_1533cd7b316d4d9eb92096b5e5a9c7c9", "dsid_d67523447eb749a8b0762bce0f4e4459", "dsid_a41808a540d54032a161310ed6e9d843" ] }, { "question_id": "qst_0389::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1d53445cc74b4d7db325cd585736f87f", "dsid_4866ef84859a43b89d77178c28d9513c", "dsid_48ba75d82cbb495aa94c71edeb3384df", "dsid_4688847e3fc04d57afbbc92d03e6f497", "dsid_456e37c950de48b49ec504feff97964e", "dsid_a9d291a78fc749758e972678d5f667e6", "dsid_a5a5fbc9d01a4ca396d77b2a5212e572", "dsid_7cf7259bd99a4ac588aacd521fd1600c", "dsid_784e529eaa494f65a3e73441506250d7", "dsid_ef4f091e952647249d7a4dcffe687e23" ] }, { "question_id": "qst_0045::metadata", "question_type": "metadata", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f14dd150be81453e9b174a68967e3afc", "dsid_9ee1a08a0c9b46e3a15efbe7d6b3b3a9", "dsid_80a4859760ce4cf4af51f7c570903ab8", "dsid_697554a81df144b5a25451c72257f5e7", "dsid_86e6f9faf9924feaac3f5a88b1a04dbd", "dsid_8aa30532ab3f4ed5adfb7e9ef0c9e808", "dsid_c7cdeae220704e28babfb27d8c19f8bb", "dsid_a94ae5ae140f4127b1b19a94b46ac072", "dsid_eab258523e1a4b23a951c5063f9319c6", "dsid_bd8cc2184b3d4bbfa359c47798c97594" ] }, { "question_id": "qst_0415::conflicting_info", "question_type": "conflicting_info", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_f106118d8ef045c7ba836a0aa8fc15b4", "dsid_a60c963f454e426da3b193511e521742", "dsid_05841423a55e4c35abd276d9899b7134", "dsid_7cb7a20999fa4fe78788c539c19c8952", "dsid_06f25d73e5d149d480404038db91cf75", "dsid_2f9c99fc8ec74495b4b08c49a6083ba8", "dsid_5dcdefc0f60c4113940f84aa783219cc", "dsid_9325c2398e0943798f7bca20772e4c9a", "dsid_e25d99ca756b49f1966ccd271db58f99", "dsid_1ccf18e34c0b48cdaecca9d65b7dd2ef" ] }, { "question_id": "qst_0045::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a673ed031ece43c5bc4a0d27a284c8dc", "dsid_bd43557752194726821d5a36d8a06da8", "dsid_a5af798449154b57aa7fd07b2193b573", "dsid_8fa2d2643c434a42abb38489242afbb7", "dsid_e5f976c021f64ad6bd4f99bf870458a4", "dsid_33b8ef3fa0134aec892debb7dcaae434", "dsid_dd07786dbc3a4faf951aad5a483e9101", "dsid_51e614e6e60844b4a83af4f9d845f8cf", "dsid_e19edcd9f42d4fc3a2df44576f649cca", "dsid_3ed876844ccd4cce8f42d3e31c96b015" ] }, { "question_id": "qst_0208::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_0378cb48015d438fac7668e261750bd7", "dsid_b545b43f999a4c31be43572bfc5bfcf4", "dsid_0cea8c4e421c4cc8819b469b853dbbd4", "dsid_9a6039c08025478a97b8df7c97155252", "dsid_19d3a824daca4e08b6c2caeb072e3cbb", "dsid_6a0c0c31eac240cb9b1fcbad57c0066a", "dsid_d8ef09a4a2594aabbfacb056619f1f90", "dsid_2b9465a92e1b422989407ef39ed7c4c1", "dsid_e39dab0b5edc4dcd827d3271ca4b5c25", "dsid_11118f2554fd46d38db5ee48f688ddd3" ] }, { "question_id": "qst_0278::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_43dbaa6340ba4f6b8f5bea0fe2354bc7", "dsid_f6f389b9e1d14fb7a84b94004494b3a0", "dsid_41177e0cc51840dc84ca0ad13ac42486", "dsid_fc5a193a6d8f4e799bf912e22c448082", "dsid_63f6f5edf1cb4ed4bd6ece9fe955bfbe", "dsid_facf9ec08cbf44f3b69abb00a61510b4", "dsid_b999f7279bd149038d7da011959f6962", "dsid_9111c0e7f686447fa290ef725db581ef", "dsid_5034e5e2a278441dbfea7fbfcd557b58", "dsid_fea7e5cc87094a2887e7bfeea907d691" ] }, { "question_id": "qst_0189::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_62902788debc4ccf96fe587a3d0d0fbc", "dsid_bbd44554aeb34c2b811c78c1b7e382c1", "dsid_8197be9fdcd74849922b03acf7b1ea7e", "dsid_20ed9182867145709feac129172377fa", "dsid_d5f59153d8fa4df8a43f66c4b1bfae6a", "dsid_0ddfeadf48294af1835098a103a4c9fb", "dsid_e515e16fbeac4e129b662d5458b1e8dd", "dsid_6f1b29eb4a554538826cd7bab6b4037e", "dsid_168fb762b10c4b77896d7821e4328094", "dsid_20a28d3d9d8c403e8772a16e4cee4c96" ] }, { "question_id": "qst_0078::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_cfc93aaf69bf456cb669f5b99c09b7fa", "dsid_b9e31a2358c7454ab25d7b424ee8284f", "dsid_fff8b5690dda4d7989b20588972c5f86", "dsid_cc4c5c2310714fd99962514beac82e74", "dsid_f9315673354f409b85626ebf4694005d", "dsid_d795682a554a4c6f85c0769bca56306d", "dsid_edd9e60a5f7e4f3eb0835259beab643b", "dsid_9110429050274939a76402ef2e91a53b", "dsid_1f5e7e23bd76477d9c07211bf61feb39", "dsid_51a48580557d4614ae473fcfca3fd0fd" ] }, { "question_id": "qst_0318::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_521553a072244d199d06a34dd6e1aca0", "dsid_d8cbe3c373104b91879d2da05b7bcd6d", "dsid_0aa853d7800b4ed7aa2779e9eae97c63", "dsid_681485e4afe04afe994f922b21517983", "dsid_d363f62f0c84438b8ed697891caf6471", "dsid_51fd2c83dfae4af39a913d2cd07bc369", "dsid_d1c65a017466400e9a891bd531b87874", "dsid_3a9ce3f606694773b3044a556819cb31", "dsid_1b20408d6e614d6aa8e134f5bb23ce98", "dsid_e92b657f618445a082d758b61408847d" ] }, { "question_id": "qst_0003::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3fd6af404fae48e6b8ea5a57875ef78f", "dsid_f89cfa7dd9214a499571192e1d28b2c8", "dsid_7906807435134884947df36ddf604287", "dsid_26515afce6ab4f9db8aac4ff0221d696", "dsid_1041d6525dec423abeee3ad63294f896", "dsid_6a5c2df52f354d3fa1ebe2e91b20af95", "dsid_91fa0b70bce1496887a3af65f040cd8a", "dsid_2afafa52cbd64ae1932b2933d43758fa", "dsid_0502e6e42a7d4d978be671fbf829c2c2", "dsid_346cbd021f94425d8e77d94806e8f054" ] }, { "question_id": "qst_0197::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c9cc1fb8593846a4b42ddad7f5cb8c99", "dsid_f1f8c246c42d4b9e8030b6320d6fb0ad", "dsid_6e2ea493db354c56bc26050e00b2dd2a", "dsid_a559687e57f24826b4beecfcd6322d48", "dsid_c03c9a3a983d4fe88ea4e2054c9e9fa3", "dsid_ceb5472c6ffb4fa89a7a0a76f6d2c7cf", "dsid_331099dcdb6346efa20bdec9f477fbd5", "dsid_2401e6c7889b4c2081c66dae551f434a", "dsid_158f9bbb596d4f9aa267d5ff4f24513b", "dsid_ebad841acbc545269fc46466bc68825e" ] }, { "question_id": "qst_0041::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_8ac31912d9974012a887f1ffa5c48ac9", "dsid_46f929172d9648d09066b0f9b4069bbf", "dsid_3e8cf249680a46a58bddd1faa893c1f4", "dsid_b9feabcc616a441dadb2e6237166b316", "dsid_1134a9b22bdb40ed9c6231eb0e5e8e82", "dsid_64af704b12474fe9a6081358d278da0c", "dsid_60052d83b00248a4a504c60e5338a10f", "dsid_5ce729ed17fa4403b2384f83c5c00a30", "dsid_6ded6cb1fa2d4dfeb489e0e9e3095d16", "dsid_cd4ba594b11e46689b1f3ead31182fad" ] }, { "question_id": "qst_0166::basic", "question_type": "basic", "rank": 10, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.1, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a110cebd6e8844afbb800d73e451bf55", "dsid_d47b69665d364df58ca39f83784544bd", "dsid_edb28b72310749718834d4e9f7083bcc", "dsid_298e706b9e2549b9a33776ebfaadecd6", "dsid_29a2eb3d8d974d18b4560fff32f361f8", "dsid_5be846b5dc5247348dc33a60aa9b0b30", "dsid_3ebb269f9c3841fdbdd14c376311e299", "dsid_4fb433d5bc984ee4b7f8a0815bb693b2", "dsid_c4325c22f35b44deb2d5df61da0b57bd", "dsid_10d018b9cd334ababafa960517492a77" ] }, { "question_id": "qst_0251::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_5c77351359f64500a0fcf0d558dc1064", "dsid_7c73ae54dc804f99b5779bb40d20b6b9", "dsid_25b3d349510d453e90b9d34d541ee27c", "dsid_da11ce26d19a468fad84b3658b20947b", "dsid_3f967a909e6d46f79a08d05e8b927917", "dsid_9800d7f35ee94aae94ad8a650980312e", "dsid_a4c50b576c7b49c08f35287731ee7857", "dsid_bef28797de8548158580d7cd328cabb5", "dsid_30dcf3c1d6c74537ba7166f35111449e", "dsid_e9a1cbf4cedb477688eec3dd9cf04893" ] }, { "question_id": "qst_0085::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2271ae6c393d409093b11b5a6d5b96ed", "dsid_c94d48a530b4477f99630e9e10b54703", "dsid_523df3a44bee447887707c0aa1f72996", "dsid_dbacebbfca96474a91249e74e54605be", "dsid_f7785a67900d432584de232f2a309040", "dsid_d9f77f614f174eeab9e2e48dcc859b5a", "dsid_602465147b7547d8b72599e2d65f09bb", "dsid_e84dca82e517439c8773d80f7977758a", "dsid_914da94309bf4c51841f0c27a14f7df8", "dsid_0e47679a4c5249ce944eaa59e3d67813" ] }, { "question_id": "qst_0448::completeness", "question_type": "completeness", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d8ec0649c34449d3bc48f7a43cf62372", "dsid_e51021765c694a65bd60b3d38b12db6e", "dsid_107b32582f064c0481b84fcc630e026c", "dsid_6c57c9d4e08947d6bdd35a0eb9bb7c6d", "dsid_b6fb2620d47d44e3b17b16b42f1e021f", "dsid_f7d5fea86ebc4266abc8a92f79b93565", "dsid_d1a5894a308e4737a4c1a591e962d337", "dsid_97200e6e3c2b4062a8cb3c13047a464c", "dsid_fe84ab73df544ee1a5496bb0e70cb654", "dsid_a53f5dc9ff3b4ef6ab68d4f6c0c5a2ed" ] }, { "question_id": "qst_0029::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_429267fe79c74918a26272f6c0e48f10", "dsid_24ad37cf41bd4d1190e6741cd5a87569", "dsid_18d6b7f9d2614de284aa176dabf09bf7", "dsid_ad4430eaf01243c2acae87cc61fc8553", "dsid_5a8502371e7043ffa2a49c103429e981", "dsid_1b909026c9e648988713b1726e1b21bd", "dsid_97443582bc3445c8b1050a8a4f298315", "dsid_2bc4f1bd8318400fa4df10ffed1cfb78", "dsid_90b449557e62467aae3ff778d43550fc", "dsid_21609ae9d84d4205953f6e44beba93c3" ] }, { "question_id": "qst_0349::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_21d6b99dc771418287d61444f373852e", "dsid_fe7a496e795b4baca997e655f9316fc5", "dsid_93105d9dc2624aec85d0a968d0f00370", "dsid_ada785411ed944e7bab1d8a5a0a42e8d", "dsid_a12e67fc3f9149b8bf684f87d57dbe46", "dsid_c386b381d5444b3fa0760b436bc51d44", "dsid_ce4c0bab3a7141eea2ff98ceb171eea8", "dsid_82ea454415034d17bd975d3a4ac1ecc2", "dsid_44cf2d75d70d4365914aa46947a94b1d", "dsid_e44765f3fe8f4d10b517e33f9e6f3b6f" ] }, { "question_id": "qst_0324::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_62a56f85dced458583381347c3242901", "dsid_a92fe998fc534d57a1d54d5a6294c938", "dsid_d8c77626e9eb4b01a4342d68c3d5603d", "dsid_3865bcbca3404c6f8f2687268d3ae93d", "dsid_1fb0af55bc3e4ef0ac1efab005de901e", "dsid_56b6c392ca654a9d98ea806b5f70ac57", "dsid_5ecd9c43cf444562a451a743d211092f", "dsid_1de4e6ce50454cd09a5bf983ed1569ff", "dsid_58a3334c0be544f681a49e69c21ff91c", "dsid_c223bf5e7c524d409e256796a2a039b9" ] }, { "question_id": "qst_0270::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7846fa5de11042acaa1147d60d58ebe9", "dsid_33de625fcfc045dd967dbc9bdde3a8b4", "dsid_0fc532b7ebb64496ae1732e13db3a486", "dsid_73c935afee1f44378aff0361108749c1", "dsid_498dea3edd9e4f27bedd35d6e937837c", "dsid_a201adb4ebfb4c48ae2b3643eaa14ae1", "dsid_a12e67fc3f9149b8bf684f87d57dbe46", "dsid_1f3c18778ff547c39c0ddf85eb5e1cdb", "dsid_9553118b633d450da2494825176bd309", "dsid_9a8ccf0b4f4545b5acfd43b8307e85be" ] }, { "question_id": "qst_0266::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_415b72b7d40a405e87b9671946e894d8", "dsid_c1f7f945d5c940f5ba28e8030ecfa720", "dsid_7bc65118ebef4a46951fd8c1ed24248c", "dsid_e88374ee23994a409f5d9947d2321fa4", "dsid_ac5e40129fa2444eb1ca1c7aa057bfe2", "dsid_88b39d13bd584ef8b1422d76994808c7", "dsid_d849fc50d6694f32b6cd04adaff35fec", "dsid_bdabd4d1a5ae43d1b53e63583847d50b", "dsid_ed4b40d5f4f746828b3644a44a9c82ed", "dsid_76420df594544d2798f9d2dca3bfd1f6" ] }, { "question_id": "qst_0247::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2167222a219042b89154a13bd7903328", "dsid_3c18e45fbc064bdeb056d94d621846f2", "dsid_f2683c74e5d949958b6c04aa9bd2d706", "dsid_ab8638b600884c9dbac0c7ae5ba2560c", "dsid_e9548ab66e4346c9a328b8990492b8f6", "dsid_46304695ee8144c7b48e2393510ff03a", "dsid_05263ad3117c441ea9fcba34b9ce3fbb", "dsid_0c06ac6db23c49dfbc8ad8049c40ff89", "dsid_67ddf2c521c042ebb5f3bdfdc139d6df", "dsid_46b9d5f77f7c40999560c010518b8e77" ] }, { "question_id": "qst_0239::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_caedab050bc54b8fb63f187b5c9ec380", "dsid_2b1e02466b1e4dbaa5bf6fe765c0cad4", "dsid_fdd27747f27949f89a9f84fe6c2aac59", "dsid_843d5646b5a543039c500c21a5e0c6f0", "dsid_1ab6c34e57574d8e9c70055e5bab1325", "dsid_3260fe01c07a4377a859ad3535073a45", "dsid_f3f5dd4e8b37472da09bd1ab0bee00b4", "dsid_d1a69931842b4b5da09e75d5e4ef45b7", "dsid_a7c5007003de495ca98115b415706d58", "dsid_59839f93573d4a0c9c98e526df581231" ] }, { "question_id": "qst_0381::constrained", "question_type": "constrained", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_aa2276f167c64dacb50aa912b2e72794", "dsid_865449ae95b044768f33773263bfeaad", "dsid_d844cc30147b49c69ea5275b4993b571", "dsid_ab14ca7029e344999d9e832d24939cae", "dsid_c0c93abadf4c438189e23cb45256c6e9", "dsid_04859af055c1408aadba967f90ee0cfc", "dsid_04587ae62d8147a6bb449869e2d0ad55", "dsid_730a6d02f1c34f8c80052b35dc739d63", "dsid_9a7191d7761642c8b2553184b5664677", "dsid_eb646fc0f26d478fbe1228c403ee866d" ] }, { "question_id": "qst_0393::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_90218d9a9af84072bcf3ff7bb90b5fa1", "dsid_4b15b253ca294377a12d12651ada2168", "dsid_7fc02b1c6fe446abb194bd51e7858b98", "dsid_bd728fb725cd4d2bbad33eb2eb1109a7", "dsid_65544d4851e74bfca62998f699271db0", "dsid_fbdb90b68a2c47369bb5011328e67ec2", "dsid_ff7f4962ebbc45b093322dd70b17af4f", "dsid_03ade157e5b9464ab16740d676f86147", "dsid_0064dac9b76e43ada0e9acf1745a20fa", "dsid_855e90efc08b41c1970978519ae3838c" ] }, { "question_id": "qst_0023::metadata", "question_type": "metadata", "rank": 10, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.1, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_576b502053d74913a254e16a4e8b7ad6", "dsid_be15e983940742569d406039294dc7f4", "dsid_318ff08b63c8402796be3c09a8d65d58", "dsid_960387549eb2472284385010b70a91ca", "dsid_92543b5ba7654cac9dc1bbcf0e43c7a0", "dsid_f91bde5f73264d78b1cfaf05c6025d99", "dsid_f02853694af44d78b6499a3347e8e6cd", "dsid_c31d2fb79c7246d3b709e788d61e72ce", "dsid_04e98bd61eac4ec89e3fc13c38de1b8e", "dsid_0a6c6e67d89c4cb4b053a7c04d780d97" ] }, { "question_id": "qst_0380::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_be8d2802cf3245088a7e7f861f317b31", "dsid_2bea8116212844eca26c9c90052b0ec2", "dsid_f1a3e4d11e274a22b0d5e646ad0ba9f8", "dsid_3e7cf5be42d84f2880858c726b65998f", "dsid_aaaf6ae3258d4fe2a6883f2cc58e7360", "dsid_089917448e654911bd186650ed12feb9", "dsid_7fb28e1d5b2944bdaa45ae8894eb8729", "dsid_b08cb0b4a55d419d933450cf8d457928", "dsid_09b15c8027114823b855e6d135df83f3", "dsid_cd7be68e755f478ab32333eb90c8e5a4" ] }, { "question_id": "qst_0124::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_30a16502953a4088b59266c8990d52d6", "dsid_b7f556eb20b74d33a653ae7d7e605d79", "dsid_fc01b9470b5b42d4a55c230b13384635", "dsid_3e55ed36d4fd48a389f5291a3d1a4a11", "dsid_3bda11c47af140e891575b02c7c9433f", "dsid_0e7bb98b6b7741a1ba20982e6d607433", "dsid_8f2545112daf4da19f1d46b3a892aef2", "dsid_04a97bccae6345f08443c01919081b42", "dsid_d1519b36a74b4ca5951d0167becbecd8", "dsid_404576fd3e304cc1aca3f91db6b449ce" ] }, { "question_id": "qst_0144::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_1a4f379bd75e484a9d28194d2b643179", "dsid_0a3977d8e5004edd9bbb851daa884bab", "dsid_f792e2a148d84e9cac998b2cb5d452f6", "dsid_87e5cd75fb97443ab381d440e955d870", "dsid_8851fbc3493042198261aabc3f7c1fc3", "dsid_5740f94238d242f4aee4ee319332ee4e", "dsid_590eec329ad3466bb0b585f4e7599f2e", "dsid_8e886700037a4f9f82c988b18ec83366", "dsid_17c39b7568b04024a0c888429a6a5c3e", "dsid_92ec68ca017444b5b519138e83233e86" ] }, { "question_id": "qst_0444::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_117e5a150aa94adc86295786bd6b7843", "dsid_5cde26f232454906879617dab1800fae", "dsid_40471ba4cd3b430eab3ac0877eef0972", "dsid_2ad49b09e5454bc1a2139c67a66c56b8", "dsid_8a7fdc9812b5489f8152526545e444ce", "dsid_50339fa427954f27a8802fe110d29ff1", "dsid_929d418a4a4e499d8b979d10df303443", "dsid_9e568f192e8b463aa5a06b6ba768f994", "dsid_4119122d1e60470cb3a9939cefb754e2", "dsid_16ce454ca380449494d7bdcd73eb4e7c" ] }, { "question_id": "qst_0359::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.8333333333333334, "retrieved_doc_ids": [ "dsid_617d538c79334f4b9ddfd5c50c05ee4b", "dsid_d67b4ea2d8834799ae617b1ccddf1f6e", "dsid_5b0c65ca162a43a0938bebabe9b1d53f", "dsid_eae4cd799908469e9c4cfc855897cd06", "dsid_2d0330be987e4163bfd9c42c69bc1e40", "dsid_ae455bffe226412c9973991ef2d34a72", "dsid_5d66d8f4c1324c03ae3cff64e6f24a94", "dsid_1b86b0fd2e4d4d40b1fbb9086d637ce2", "dsid_3e88e991c1184d01b057f6ef5da15cfd", "dsid_029e9c3544a54c0884054d84bcbcd9c9" ] }, { "question_id": "qst_0131::basic", "question_type": "basic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_24f6523c83ee49b8b16bad04d091f7af", "dsid_1d3bc5d7f88642b5a78db466aa6b4b4e", "dsid_e95b1d81126d470ea72e5ee55e454eb4", "dsid_c1d04172306641e4a40ee30354fcc67e", "dsid_b435e86904a442cf82514770beadfed9", "dsid_e9dd560753924c1e8438cdf9c2afcf68", "dsid_d9ddefc8273c499a8529d37a5d8ced75", "dsid_24529a906d5d494faeb111bcae7b6b79", "dsid_0a21d0c7e7ff4270b00aecb8cfea263f", "dsid_2a94754480884dd6a23c3080e4a98bef" ] }, { "question_id": "qst_0449::completeness", "question_type": "completeness", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 0.3333333333333333, "retrieved_doc_ids": [ "dsid_e29a9198a5754ebe85cab10c9416e937", "dsid_5cf1a6657f1848968b2fd154066950ae", "dsid_ed6d086c9faf4272b0ee20d3fa4b6180", "dsid_8dff963a50774cdd84131d9ae5e4f4f3", "dsid_f1b7863b01c64a7bb49f8b8cab133200", "dsid_9a7191d7761642c8b2553184b5664677", "dsid_4dbbd24face84628ba3db51f59e9ec12", "dsid_232941fdc7e74d1d83241c77144d0ad6", "dsid_085f129aba024b7bbde62ed88ff1f9ec", "dsid_fe6c423c71f04547b10f617e4a01af33" ] }, { "question_id": "qst_0162::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a3fdfc45905f46f3988ad13f55ecce93", "dsid_c9cb551fa9874c0db378a9936017740b", "dsid_3042eea9b36942ae8084eab2f28c3012", "dsid_5053cc95922a48d08b31014fe7ff202b", "dsid_f3539272263d4896a2f5ade8725b11e8", "dsid_a18bf0f985504c4d909ea45becc7b11d", "dsid_3e413744bfcb4f2082871da734e6f96b", "dsid_0e562e8a0f5a45af8d00b0a6c4d886cd", "dsid_41d08aec62844f0683786ff59933b1f4", "dsid_6a5cb17241324ba7ab59d4e239a76d60" ] }, { "question_id": "qst_0034::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1867cdf06a00437194b3250c75321a92", "dsid_85d3f7f2294c47d4b26b214fde58063a", "dsid_634dd9a56b8c4d4386c4f87b53ac16b8", "dsid_dd03b7ca67494e3d9d6c34bba9d027c3", "dsid_7cf7e11b5e724b2aa36ad8575cdfa226", "dsid_b77fd838edd94ccaa3066491bff54b4e", "dsid_2c0124487e2640bf8cbd4677b9e670b9", "dsid_7f5d87ade6f04866b93a4e60f50e8748", "dsid_8052c3a6f6204ef99802017a8e30dd2b", "dsid_4dca1233f44a48ea82d0f7a50e298463" ] }, { "question_id": "qst_0238::semantic", "question_type": "semantic", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_81ea322b99b64268ac6f98e4f562c848", "dsid_da92e1ed187e419bbc926fa122e2a7cd", "dsid_7a25e9b6949b4b7da4f050661f626990", "dsid_92f56ed27b75455f82f43528f6644f3c", "dsid_633da4d2b9694c60b5a5120fa21c8652", "dsid_234e2143e0a94f4185df35378440ba43", "dsid_0636d2afd19944b3ab8860096710b91a", "dsid_e528bed025b44f71bc37a54f5a9aefcd", "dsid_a5a8f859c9134ef9827a7d674b020eeb", "dsid_98b3384717b549f9a40d114f0cd6cd71" ] }, { "question_id": "qst_0057::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_026bbc0fbb97423ab9de20d8f99402d6", "dsid_d740645aa8ce42059a6967bdad866b82", "dsid_718a5c08e1ef47d38634c3623ace0803", "dsid_3c368798ac95482a8bef9e28b6f4af96", "dsid_4688da04d8b549e68271991245290a3b", "dsid_e9c5246734c849a899e29c60e6134988", "dsid_ea08c39b16de4b76bf74ef89523333f2", "dsid_3247b6d0a4e84fa6accf2789b28efa23", "dsid_5cf28151bea84f5b9736145cc853cf83", "dsid_54628cf1597541c2878708e1c2e511c4" ] }, { "question_id": "qst_0268::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_46b0bc1e97ae4d8792d08e6f4122b948", "dsid_d4319ac5979642dfb0612c4f35cd4668", "dsid_a990e708840f481795741cfd3fb55691", "dsid_9c62c96a8d3c4a4fa67b4b61d58b1806", "dsid_32d162435ad34eeca3861eeca4463879", "dsid_d885f7fe13ec469d8b6ac95d64c2fcec", "dsid_5b35d075a1c24243aa9e542eec17c5f2", "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_012a01961ac74fd28a829c2bb7850573", "dsid_a7baddf163ef4946a97b4765bf1cf08b" ] }, { "question_id": "qst_0261::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fa79643c4fe4430395363b868d306267", "dsid_c9a1ca32dae14498a89c2ae1b1d5bfa8", "dsid_5f137275c8ed438cb7edc9f1d8740ddf", "dsid_c6f16a3fc2e84f2597dc0786e4f16ae0", "dsid_5fcac9f223fa49b58cb327d7c32f99be", "dsid_f1ef097128ef4a1e93ccd903e10642e8", "dsid_1262215cc2c54867b2764c48d64d84e0", "dsid_03bbbaa2fd64496395b8f6991372bcc4", "dsid_a2f59c7fc7254d79bfb345caa3207a86", "dsid_fb109b3b06cc4aa4b5db05949fd66e17" ] }, { "question_id": "qst_0021::metadata", "question_type": "metadata", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0a3977d8e5004edd9bbb851daa884bab", "dsid_929598314f834ad08cf48642b88bde16", "dsid_ddbe235d0a69489fae2809702b2e6b24", "dsid_b62f660578c64e7daae4baa0c6b64e8b", "dsid_e31c9f4814284767820cabdd7ea7774d", "dsid_d05cb3bab0b049c1a0089f3b187ac890", "dsid_9c3ba95c23c744ceb19c493f0e9f99bf", "dsid_78da2a18323142f2a3fbd2a310a1c9ec", "dsid_a1ec870e3bee475cb31ef189b5430b59", "dsid_dc96257dfb474c3a9c0d086765c039ea" ] }, { "question_id": "qst_0298::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_94fff4908fb14a8eac5eafc25fbe7f6b", "dsid_e8e41ea4c8af47f4818fdda98ec43175", "dsid_9d471c0660bf401da9fcbd222bd889aa", "dsid_408c94bb372743f4a622da1d30717c83", "dsid_e787c48ea74e43968a6dbb4674d4db86", "dsid_719f6715eec6487ab8a92f88f8d83257", "dsid_fdcb08fb42594a2ebec044cf1183c6fe", "dsid_bb8ea6cd6dc3405c8b3560b33a1f50cc", "dsid_c0e5cd21be0c4677ad0aaf5494aec363", "dsid_9c6b47a5bc944951ba712393e3f0c864" ] }, { "question_id": "qst_0313::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_de02efca4c804ed8a31dacc77a73b769", "dsid_e9f22ba025e34bb6b7b8c61d26a38a19", "dsid_30ce3339ae2c41e8b5011ae8e824ef8c", "dsid_7f2a6f2d2de34f94a5b6f3d5d0d1f6a2", "dsid_0f789284a1514713b7fd67f4d810d258", "dsid_b4f5ca2611ec473c8affea7e0fcfe4bd", "dsid_87fb9790b17347268d4e0672af499c40", "dsid_72f9a72f30184cb8b1093389c0c6a8d4", "dsid_1988feff8a414541a33ba3ac70400ac4", "dsid_d6d2b4ba1a7549a0861c4f3614ee07ad" ] }, { "question_id": "qst_0244::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ca15f318846949608e88005cc49a5a8f", "dsid_37c24c697d9d462fad0e3afeeea9d8c0", "dsid_3c784dddf8254494b0355227acfd14e8", "dsid_85cd5df03df744d5811b1f0a388c4652", "dsid_8e85c02011ff4ddc92ff86bcfa3cb72d", "dsid_3df48dcb80af4806a88d4ed0f9b04c30", "dsid_42e93906357541fd97d56f5f20923a9f", "dsid_c3ab94fe217d4dfe9c6e0b88a40cb6dd", "dsid_372fa164f5814ebe9adffb584600d9b4", "dsid_af4858ff547a4246a84e385db1c21aa8" ] }, { "question_id": "qst_0077::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1aa1420618c0424eb1ccdd38fce758d4", "dsid_1dffa4bc09594854ba264a78145d4dd1", "dsid_88b39d13bd584ef8b1422d76994808c7", "dsid_b777b0c6d20a47a698ab7c3c2b92be94", "dsid_169dca195c96410b80ff166803a50512", "dsid_c700c183b2794ecbbd17b7b42d6bdb9d", "dsid_64ace862b7c041298dd4830a661cc4bc", "dsid_322d675b5d82460b9bbbef52704d2b7f", "dsid_90ae4615ed9840e1966dedf508cecf79", "dsid_2c4c661e9f4741feb5dbc8e0817ae749" ] }, { "question_id": "qst_0424::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d9f77f614f174eeab9e2e48dcc859b5a", "dsid_d43cbff0391944e8b00d75141767ff98", "dsid_1f5f10cbdf6b464c8c1cac1f4bb48e27", "dsid_403e41101d1d43fabcd3f392ad1afae8", "dsid_f34f0f2a273d4c18b06afc7a77fdf58d", "dsid_7f6f5f0ab0e2481f8c3b03d69df3f3af", "dsid_4101bfe0e39e47208ebbacfdc741f643", "dsid_904e9c6d4fe3443fb66e7f7eccb4d782", "dsid_5e2b017deb9d4b8e8fa422b37964bfc5", "dsid_c4e1081cf0f0451eb51ae121de0ec3a2" ] }, { "question_id": "qst_0401::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f7dc91cc70e14c94a6215a13cd829cbf", "dsid_d36eb4801f674ebd8e9a44fb730b7555", "dsid_80df3a989d0e417baec286cd4c0d3e61", "dsid_f5545e84d0af43e38eebc77467015394", "dsid_b1280f6db9fa45dd94b330b245f49743", "dsid_1e6ab42febf340e19f01361bda5b0243", "dsid_c1980a818cd8454a8ceb5a5469830d05", "dsid_b42ba4eaf9cb4db697bde90e61b8becc", "dsid_c11c3a1f707f4239a46aa647faa01230", "dsid_a84ce4f899f244d092e65bb505ef8362" ] }, { "question_id": "qst_0053::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_09db191ac38843c5a6ee7d4b33dfcee8", "dsid_36ffc3c8a15a4e5fa4ccebfdf00848d9", "dsid_6e6f6157c7024ca79662118d3c647f36", "dsid_1ce4107e10624d2e8e84cd533bbd2ad4", "dsid_0dc68466ec674615a70764c82974a669", "dsid_ddcc9309157b44978465c4a6b8b4c9bc", "dsid_c3a30d817be140188bc7569ed5f7ef34", "dsid_aeaed65cdd3741409ec8bc5d0d2b9dff", "dsid_933db1d0bd9844f7ada03d735230a7d1", "dsid_cd828310278b41aebc9f6f7f57b9d605" ] } ] }, { "corpus_scale_size": 100000, "condition": "raw", "hit_at_1": 0.385, "hit_at_5": 0.565, "hit_at_10": 0.61, "mrr": 0.465, "document_recall": 0.5769, "per_question": [ { "question_id": "qst_0102::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e044274a9d184700bb8cdf9ddb6d9d02", "dsid_540607d488c4447baa8f6f1408ec42bb", "dsid_8e27ea15627346d4ae210a5ac28bd2a2", "dsid_0b98dd77a056407aac139a12a16302de", "dsid_2bb75fde3be24cfcbb35ecb1c14dab21", "dsid_639dd0469ca34277a35642bf3a09d3e9", "dsid_1c163e07a1ba45e7b81e265a20565c36", "dsid_af125e875c004628a365e906c6d889ea", "dsid_bd4cfd315c2143a0a1199056b9bc35be", "dsid_c2effe60c1f74d73b64dbdc05ba9b9b9" ] }, { "question_id": "qst_0002::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ae3a5380355d4db983aad19c6939b26f", "dsid_84c11781f8b6429b9980acd48e9a1fc5", "dsid_c1b16f241ebf46caafdd845edd2c3da9", "dsid_8eaa2785974140b3894297324977a8ff", "dsid_29bd519990504d7abf163cc4fa6ca4ff", "dsid_90dca687911e4baeb9ad961be8f0da4f", "dsid_4b2ac6984bdf4737a18520e158b04453", "dsid_260f077d25b04fa1be926188e8cb20b8", "dsid_cb698ab546854ee58b83043017ac0790", "dsid_6ddbb46c266647f4922b8107fd6ea395" ] }, { "question_id": "qst_0252::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_15e23a53f5124b67ac74eec12a9c59a0", "dsid_6f7992198948439fba3b7194cb3f8b4a", "dsid_5e5365d296024949a26033b26fa169d1", "dsid_7a1c8832d9aa4a2cb830f4824ca8acc7", "dsid_6e2ea493db354c56bc26050e00b2dd2a", "dsid_7b4d5aae7f064e25aa4e0138a6c9d4d0", "dsid_b83c94333b424640a913e66e59433410", "dsid_a5b2109e90b14ba6ac841ff8dec11073", "dsid_4cd70ed8e40947e2a080a0715c4faaed", "dsid_d8e75effbe37456183c936ee38d3a302" ] }, { "question_id": "qst_0020::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1c53affffb4f477ca476bec3138b60eb", "dsid_87fd5087a63444fda38fcdc8dbd90ee0", "dsid_22b0fa396e0c447ab845a50dd70f59ed", "dsid_292bf5b28c4041ff80ddad285249c4b7", "dsid_794fdb93c46c4857aa923252fad352e4", "dsid_3abfb1b3210d47009ff9dd0f1aa6ec86", "dsid_321f050a1173469fb4a461b5cd96d170", "dsid_c4e18f4f1d0241508865776be8a0fa01", "dsid_4a9c5de4f9f145cfb9cc9a3b3219db42", "dsid_4b4ff93a17874621b8e10de846cc17ce" ] }, { "question_id": "qst_0113::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c53d4b486a1e485186a7e4b9d1c3f3ef", "dsid_b53c41db3d79493e9873acbc8340a46f", "dsid_db9f6f36e1e14736a36e4282ae5ca113", "dsid_3812d098cb404e3ba3d4ecf51436271c", "dsid_ad80bfda23ee4c49a6b7ff5ea1beca7d", "dsid_574c6f4f460d4da893026879eb4eb3b3", "dsid_9efefda54cb04cf7aa400bf8f5e98777", "dsid_5c436a5958e14b55b136388b0b06ae4a", "dsid_f8f7b2e0fc6743e586aff9a83f496282", "dsid_7400bdf63ef74358a0c1df8de4b5a4df" ] }, { "question_id": "qst_0204::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c2870779c758424eb1af2fe2aa3038ec", "dsid_c46cce673408472691bfc450aae03ee1", "dsid_b1dfafc3617445f9860c58dfb7d779a8", "dsid_97789217db10407e876d7ccc080ec1c2", "dsid_d0660866e36b4877b1f828b395181c9c", "dsid_5f002af739e44a258a84429d007bd5a2", "dsid_9b7ece0c1a544cbd91357f676aee67ab", "dsid_f94063a7a9694d0cad2d3e4f70222178", "dsid_9a7a399188d2485f866e95dc2c7e82bc", "dsid_f63fe60d63a646b4a6c6a78eb107ed35" ] }, { "question_id": "qst_0089::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_bdfebca7da6c48b7a29ead4fddaa79ae", "dsid_ff16172c98f44e4f8c4b29dd3af2b734", "dsid_368c2e67e1874131a10795d8e5de5827", "dsid_0e5d54e456ab4b1ba32b3797b8e44ded", "dsid_57071f1894c840998c562ffeb1d2e8e8", "dsid_1400062839f747889e0b8a2a5f7080ae", "dsid_37ca232bc6664359b9468d373c8f6ed2", "dsid_c9770748e4f94e2aa63b91d342ae5241", "dsid_af565591ed904f23a2de393951faea92", "dsid_4dc4ffb08e894b698209725ba7762eb6" ] }, { "question_id": "qst_0006::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5b3278541a824fc7a21d24dc2d7bd5ba", "dsid_184be937d34a412ab5e61366d54d8ed6", "dsid_6d68900294db4ea59045e0aecea0618e", "dsid_6b527c37489745d9957f6af2e15b7fc0", "dsid_adf4b0636870464f9fd379dcab147388", "dsid_4f2844ba27d9489d882e1a332db2c618", "dsid_b9461f25168442e1a9259b8b401ff946", "dsid_bf84fe7e5b12437580f1b235fdcaf2c4", "dsid_f60a7530d68b40c3bcadfa26f42e48f6", "dsid_c61e9969f74f4979b6814e128a966a54" ] }, { "question_id": "qst_0106::basic", "question_type": "basic", "rank": 10, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.1, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d6ca2f261ef548b792cb7ac9a2d3f7d5", "dsid_21e26c2e505e40e38f52a9d4bd906576", "dsid_47aefe3cb5bd4cb196da8b8c0d5beafb", "dsid_0c8136b53ede406a8ec1c546dc132d4c", "dsid_e0907fe67af84eae9110a4ed5f97557e", "dsid_4f7318bf2d7f443c8bfd3eb99c228101", "dsid_5103675c329f4b9bb921bf53f3944e5e", "dsid_c56494f96ed34157979360e408bf5238", "dsid_2cf26263265543ca8be5f1267a7d1142", "dsid_6f69a170c5ab486588cf0bd92e0d53db" ] }, { "question_id": "qst_0269::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fbbdefae0e6b4f80bd50166c0f2395a7", "dsid_fad07fd45c964078ba757dfd8d79ee51", "dsid_ae69ce3fd65f43c0992b91f74278b76b", "dsid_e6a18f56aa6c49b0a39ed37db62c8bfd", "dsid_b26d0b4e645444beb0be8b7c53cbdca0", "dsid_aa32fc5e084a4a35983bebb576693b7f", "dsid_12ac52f659184a40a95c762d76b110b9", "dsid_5e087d4f8c8b4a238c98bbfd9bdd7a50", "dsid_f466dfe813bf466580aec7b0b691fc42", "dsid_e0e84f2aab1d470cac591601cf16ed44" ] }, { "question_id": "qst_0025::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b545f0e016d54013808c7e36145d287c", "dsid_012a01961ac74fd28a829c2bb7850573", "dsid_8785355d8ff74c8290125bd341d782c0", "dsid_8ce42ae1c594418992c06b739d364614", "dsid_8401a32f4d644ecab18ee24ad6149873", "dsid_7b3e01f4f23a4015b711a21f55dee615", "dsid_01c7aa20e6f447a89af72eee77aea203", "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_f78be6e8423e47a497acece99aee2cac", "dsid_af90435969d446b4a1b52b212b969e8c" ] }, { "question_id": "qst_0054::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_44306557500243dfaad02c028906c015", "dsid_dcc34f102c3949dcbed77d531669e310", "dsid_bea915b9aaab42e88c5b1cd53370d90c", "dsid_4210fbb4f63c404c837bc9c714c55a99", "dsid_c64c29822dbf4194947535359c0ba77c", "dsid_17bb4c84941e4fde82a79a0b1099c0da", "dsid_b9f1385353614254aca460da1bfcccf2", "dsid_b8558b8d57494d1ca36ca54d75b4f538", "dsid_6aedd6f3280940acb11f7e21cd658e7c", "dsid_be17b5b3b0db4734bb5dd933cfa959ad" ] }, { "question_id": "qst_0048::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f3af4a0db68c4b15983439460cc73006", "dsid_2a4d14cb1f7142a9a291d2801a53259d", "dsid_3cfa5c8125984eaea2377973bf53f012", "dsid_7a941ff5fc3a435892c8e254cfeb3c57", "dsid_d7e3daf290784c798806f71af4746e2f", "dsid_9c179646fb714fdd9dd272b19ae75ff8", "dsid_d6df55553ea64bdd9baa505c3340443b", "dsid_c5fc911145e74aa7947ed8858f9b8548", "dsid_0bf83a391c904c1c8798d1dfd6561d31", "dsid_a83872e693df4af9be4fd4c477273128" ] }, { "question_id": "qst_0181::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_7df11b381971496da60de2fa837ad090", "dsid_993bd7e86ff647dbb871e5e1442a7790", "dsid_d88a4b22b6ad45c583302e8fa3492a96", "dsid_ead5b36435fb4170b49ec5e9dcb07787", "dsid_2f528e76ac7243929106d218ce3822d4", "dsid_ba16d5ba48764a7cbd00c69b344d27f6", "dsid_0fd905b55f43470f99da797c9ecdb2f5", "dsid_0c7162f9c970448dbb2bca006a4f1715", "dsid_f3cf569c1eb448f39cf5c531fffbe405", "dsid_a734cb0f87894b66825518fc956445fd" ] }, { "question_id": "qst_0044::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_6fc2e1014c174e48a4f81442f6c9f007", "dsid_8107397d5e8e4a028e7f1498880f44f6", "dsid_47a19820c71f42889c8a68484496154a", "dsid_77747e21fac94ef093179dac9c6a86bf", "dsid_9b7ece0c1a544cbd91357f676aee67ab", "dsid_f63fe60d63a646b4a6c6a78eb107ed35", "dsid_e14438ba596c4446afb7166dffaea34c", "dsid_13de211883dc4e9cb2d1fff3b7828084", "dsid_f90edc9a3b30467e88b8e10955d28018", "dsid_d2995bad08a94af4ac7fba774b5f6b31" ] }, { "question_id": "qst_0275::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a55032c4c2064671a4b0abf975438e1a", "dsid_a8c85ca9a6ec4502807e537149dcfd57", "dsid_af84a556ba75430b9268f16e484310a7", "dsid_8a86457a17324b86a7d65d9882554626", "dsid_4602092f1520439081cbbd1b134a58f6", "dsid_17053c6bd4074e9cb526961d9a20b4b7", "dsid_0f34ae8e689c432d99c7dfc53d9073da", "dsid_1aa99d3657714a83929e9ecdd2a78cf6", "dsid_3e7cdcfd47de4f65b6b5a3999b25d4ef", "dsid_9b01d799e6f94ec3a832b6bfcc02a4fd" ] }, { "question_id": "qst_0317::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fb1829d221b94f98b219931b76903483", "dsid_2ba740c158c043f597f664fa1b848b27", "dsid_81694f1e00b048069ab1906212054ea6", "dsid_ddc40a1dbc264cca9276a38359d4008d", "dsid_c2a00b1e827c4ec9b43b057e2aca2215", "dsid_bd2988d48fc5483ca3d9445f2c4b5363", "dsid_bb956c4d65ca4891bdb0737bfc9e9724", "dsid_c16066922b6742eea82a028cf7ed0769", "dsid_23f0de8128b044e993b86ce02e9cdbb4", "dsid_a10a64eb557643969aadcc433e634f33" ] }, { "question_id": "qst_0017::metadata", "question_type": "metadata", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e7940a0dce1f4cc0a1b1e679e32058fa", "dsid_c83811391709468fa9534e6495c205e3", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_54c2b31af18746adaa5b715e4e92df89", "dsid_a3f48a87dc254e8081bb6f451b4c77e6", "dsid_01f4ce1ac9ca4b85a471d39960e6be37", "dsid_ddd6993b770f422285f2927dbe450cb6", "dsid_5df40cdac34b4a8f86ffcf80ec9f6c6d", "dsid_fe5b2e04db744e708cdbf3e54ea78d8e", "dsid_b59ea50687c241c692715a3111a0e9f1" ] }, { "question_id": "qst_0048::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8412db85d95a45f7bbfb14dacedb4d8b", "dsid_aa6758ff002843e4b1b46c60794fb189", "dsid_e8ba83e5c1c04b2bb7fb99d6fcbed3cb", "dsid_88424ba4c44b4f70ac7902651a78ef1c", "dsid_875211cc3422429582d22ab6387ddc2e", "dsid_23d9b80004ed423586a406dddccbe0aa", "dsid_f9e49403657647969208e34a0ebb6bb9", "dsid_0bbfd01067e9491b8cbc09e0b4a3c9a6", "dsid_e751c9e8106d4136aa3753006ad5fa31", "dsid_e7bb00f30b9e4d9287beb0b55e38910e" ] }, { "question_id": "qst_0060::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ada1b8ab101f4d3d91f8f42d98b3c693", "dsid_2e3e9c317bdd4811a56e079b1dabeb8f", "dsid_e5fe3fd0cf444668a64970ec00818f46", "dsid_9b30f5e9d84346fda6017553e8a78932", "dsid_b5aee2654e9a4f298d6d34212ed88211", "dsid_8b56f59b3a7744e3bb4b429912ad5aa2", "dsid_c2daaab889b246df8652445b3ff184fa", "dsid_328b258e0f664bb5aa545595343dc2e0", "dsid_936b93cd419c494dad4544b60527bcc9", "dsid_538be95952e64be59c342d9d2c076382" ] }, { "question_id": "qst_0209::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_70b7129b5b5940d8a17b1322f649957a", "dsid_2b7b1f23d05c47508ba435dce0558072", "dsid_5e1fc88fecea47c7a2a74bc1ef063bad", "dsid_683dc2cf9bcb41b7be76949c6e0965e6", "dsid_c960c001e9f2479e9ff9567a2c1d795f", "dsid_0592c49729a44b3e944522c6e98854aa", "dsid_33dfbd7e6dce429ab71aa69998f63284", "dsid_7e80f6e981244d988aeef5ff8b43a7e3", "dsid_1615c3fad6104b71af4e38cddaa97992", "dsid_9fad67220fcc4e6596b8f2cc292872d1" ] }, { "question_id": "qst_0310::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_db3ab5274e57416e9127f51b797b7364", "dsid_5e578186c8a6408a95f2410d2b5f48d6", "dsid_484f118b24dd4bd3b290e59dce06e8f8", "dsid_a3c04ede3ebd4ac5999e99fdf7e5392c", "dsid_ff14f394ef224c878e7e7478085fabf9", "dsid_eef176b5d0444d768358e3906e7a96a4", "dsid_5e768501ced8448fb09168b451b3e74f", "dsid_32c4c9ee86e44c799676b3ede42585e3", "dsid_de4544e04bd64f59bf7c773e44b3e391", "dsid_a5d293df3be843caa748deb7c0946abf" ] }, { "question_id": "qst_0067::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_52f229154fa844679a2173b7f3263b0c", "dsid_3703253053174363ba6c47954166f7f5", "dsid_14a447cb5b1140709c7cb3cc9a88c430", "dsid_bdd6a5e832664f20aa0fcf81e1f80fd5", "dsid_16485be3dea549d984d3d3cd1d9da017", "dsid_84be9f4da0124b79b66051481cb10ac3", "dsid_1777ab53741348ad95d480ff0a7e0049", "dsid_ae4ded37c19141a092d7b2198ab5128a", "dsid_9e06fa6720234ca6b8322938d035dd26", "dsid_ba24cefc5f9e44e8bb45cc2f3362ee66" ] }, { "question_id": "qst_0002::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_03bea1fa6f944d479b97f6c30123c252", "dsid_a73b8219c8f14af88d36515e887c9d37", "dsid_9a1b0f209a284629adc2992715369ad0", "dsid_e5307f0e837d43cbbf8fc9c954e198cf", "dsid_1fb4ac4333314be3999767e281bb4a0a", "dsid_67da84f2fa18400293050a6e178940e5", "dsid_ac17673b4827489cbc0c55136fb48d8b", "dsid_cab11c2b0b7c4d16ac74e9f4584acbf0", "dsid_f2d5557e5a66420ab69617b62d95bc79", "dsid_fd82d8ef7b0d4b55851e66bda26b9d62" ] }, { "question_id": "qst_0350::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.6666666666666666, "retrieved_doc_ids": [ "dsid_f3e6edec6f4947a5bd5aa7fd557d7d2b", "dsid_939eea1a52f84393b0f87492b9aba8c2", "dsid_e2bf084a4d924d6d923a309234cfad2b", "dsid_fd3f961d8f6a4e39a6fbc039172061c0", "dsid_8590bba128564aa7a60be6067b6be779", "dsid_e47ee65f0bdc4000b70ee9f24a01a234", "dsid_13ee8fcce6874cdd842fdfa592edf899", "dsid_92241246d2c64816a5ef8f7fc925f7da", "dsid_7319adbdfeb94cf9a955a23092938026", "dsid_51ab2311e8e9430daf68a13516455234" ] }, { "question_id": "qst_0388::constrained", "question_type": "constrained", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c37fc945bf8947bcabc9f84cc4cf1f29", "dsid_8c15ff526e00491e9ccbde5ef4a4458c", "dsid_e34a086560eb4be0a95e3d566b810226", "dsid_1258e661701942a19c65972a3ad238c2", "dsid_9479faf893494f40be6f4593b656dd73", "dsid_f78be6e8423e47a497acece99aee2cac", "dsid_68fbf6106e38493e94a832e218fa14c1", "dsid_321f18280bec4e63a05203290726e89c", "dsid_53e7bb9a21d1477b88cf83e92cb434b4", "dsid_b2c5ab151b4d4407aada768a19611ed0" ] }, { "question_id": "qst_0004::basic", "question_type": "basic", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6c2cf8145ebf447ea475bd95c7354fd3", "dsid_7d9551aa21124d70b0e247c390d88622", "dsid_04f21e239c1443a481e0e120f389e4a5", "dsid_68d0171658ef47efbc6cb7009ae7975c", "dsid_5e267e413c72438f8125a281ae7f9ad5", "dsid_45dafd73b0704a759d31eb27461b13a1", "dsid_056fb9cdc94d4d8e849f378c2a7c1118", "dsid_6c4c1c875e704f09b4d791d64d7bc7e5", "dsid_41ae740164754c88b36a015e63ee42a9", "dsid_ee82a6fb454f472cb04ac8bcd46953c2" ] }, { "question_id": "qst_0344::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.75, "retrieved_doc_ids": [ "dsid_4b2ac6984bdf4737a18520e158b04453", "dsid_409618a3958944f9b78362ed7e01bb33", "dsid_8c704516b3e94f9fa987d242dd100647", "dsid_ad6beda0999448ada51637be6063cac2", "dsid_b5d7f83a9f4d408db0b3905e8e7e7257", "dsid_a5a5235d973e421c870e11ceea4887ed", "dsid_3c5ac21ebd974af7890a852c2c873060", "dsid_535ca8426b1e421ca681b1853f340131", "dsid_401fe66f82e2440f8a5eb6f64cbb9de9", "dsid_ef048583ee2a41ceaf97142a4ad3550c" ] }, { "question_id": "qst_0170::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_14338102b4a5440fa16c105feee7a9c9", "dsid_5f05730571434ffc8cfac8e36e44928e", "dsid_9fad67220fcc4e6596b8f2cc292872d1", "dsid_ad86cb08b89e493d88aeff4fbd767384", "dsid_7713886531024e76a45cdda8aa959205", "dsid_1416af2293c64490b6aad5b1bd72fd59", "dsid_4c05f05586064798ae5e41db8873eca0", "dsid_33dfbd7e6dce429ab71aa69998f63284", "dsid_d3bd17423d90436a975a4153bb70769a", "dsid_d4b0d2fc4c334f3c877b1248675f693b" ] }, { "question_id": "qst_0213::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_314497df33bf4b41bbe8fada9cd63efe", "dsid_5e9a7a2da24d4655b587c13ee432165a", "dsid_73f607f4ddd54243bb196e7b1393b329", "dsid_2c33f36bb9254f6cac917de0f81bdc60", "dsid_f85721e1c048491e8222a383654709ad", "dsid_f9266d0ba58e4d51b088a831a3f230a6", "dsid_f560ef99b8a44495b9cdd9154129819c", "dsid_a2ae19557a1f4a35a96902967bfbed19", "dsid_8ca57db23e40423c9278dd6826d09582", "dsid_4b07ee432d54461e8e41640a184a47ef" ] }, { "question_id": "qst_0368::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_abfb68dde21f4fd2aa7d70c472447f89", "dsid_16003ee4fada406da751b9bedb93d5ce", "dsid_8761a5900a7f422e8a2606576740f2bc", "dsid_826a15051f844d6ab1c1ae9bf0b49bf2", "dsid_5ddf1c80a8514569a961768697429dda", "dsid_718e071d5b5b4f27a48d50ae5537548e", "dsid_3f2cf90e674341fda45c81220ab93eda", "dsid_f5db113b21114efe8f65c736ab8319ac", "dsid_94b0d0306935482d820e38d0bd986139", "dsid_18ac88b8d1ce4470a99abeebfca95cd4" ] }, { "question_id": "qst_0056::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_88bc012d9dc74b4f9947f88c184bd71d", "dsid_2d07ca95c76c4cbbbdde4cd7f081d3ed", "dsid_d35abd7e40cf4e55bf573c543b2c49e4", "dsid_434b1fdb94514ddf8e0a90ac30b46b21", "dsid_d8b60ea5b86a48beb6b03a6f2be0d834", "dsid_e9890131f78e42f890854e8d56efd5e0", "dsid_05a5648cf37c46a2a756232e81b3cf7c", "dsid_2cfe3f56de90412599f3c7fb1b1df829", "dsid_fe8f080f92924ae9a5ec920c956cde59", "dsid_9229aa41dea84edeb3357de2145bd30d" ] }, { "question_id": "qst_0071::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_885718f12da643f788ab132acaa86eec", "dsid_9d3d1641bd964c09af613801c922c69d", "dsid_d46e3247231240a5b6ea789e0aed7aac", "dsid_116e6d680857469d9326b2b7f1bea156", "dsid_e21dd8168db644bba2c6db2e9cf86fe4", "dsid_084339dd695340458272ab36785c08ed", "dsid_083a7efba62a46dd8d58be118142a403", "dsid_d9e7c0e0d2614a9ca91a87f83e6958e9", "dsid_4affe9870ba54d988c229686a7f819ac", "dsid_84e1e2540fed4e7da632c7644d822cd2" ] }, { "question_id": "qst_0116::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_6c57c9d4e08947d6bdd35a0eb9bb7c6d", "dsid_b579411f32ab451e9b9e1f8ec7c2310d", "dsid_38cea77f0c0c45e9a08e6b9e9bc4a41f", "dsid_e51021765c694a65bd60b3d38b12db6e", "dsid_5a060c16f9e943dfb6556b74d1bee39e", "dsid_58f0dbd88c0047d291e4846a2538f0e5", "dsid_3268b2930cfa465db7115b2c22f5612e", "dsid_3c1d24c5218f4cfdb12df85e45d3ea7d", "dsid_d8ec0649c34449d3bc48f7a43cf62372", "dsid_c356d1d687b4493f92c01b693d4e98f2" ] }, { "question_id": "qst_0227::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_42d51fc5cb7746e4b1c9200136795a36", "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_8dfe3e03d22d4e61b79b11b6012cc249", "dsid_46b0bc1e97ae4d8792d08e6f4122b948", "dsid_9b22ff67ed0e44989ba66e908c32b067", "dsid_a4b8e6b6bcc7412ca823e5795f6ccf82", "dsid_92ffa8ad67484ae99c3123765115db3b", "dsid_01a9ad7bc2624b2d81b007fe7b92cbd8", "dsid_3ea40e1fa3fc449fbf48ba6a94801808", "dsid_46383d57889e429380b4ea4d9bfb5646" ] }, { "question_id": "qst_0019::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e728a420a02a4ddcb125e6adb09d211e", "dsid_eb415ca0c9ac49208ca007c2ff7c3793", "dsid_c5c60dcb10a941f1a02ea21fdf708e46", "dsid_39923eb26df4401ba20689b392439695", "dsid_5d35e2c0250641f58eb04a8dc9020282", "dsid_f38fa8a77d194f6bb32d3045fe06a80d", "dsid_775221c79f314bcaba5bfe12df57479f", "dsid_b6921e85d3ac4bb3aaaa91c881b38628", "dsid_40774fbb68b14c5e85e20b9648e1d347", "dsid_1bcb100acdb24b43bd8f125084cc35c4" ] }, { "question_id": "qst_0047::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_52d6c2ff7ec64201a6cfe7cc2005d00b", "dsid_7a2cfe4b7e574e6995f33e64bbdbf426", "dsid_81dc9f09609f4eb9ad22b5a8fe906416", "dsid_b0ad9c8fcb6b492a9e5b49ea4a88f557", "dsid_cf989b3506d1425aa75fb226b39eeb31", "dsid_7ab12a0c235947b4a69264e27b2dcf07", "dsid_847e5271edc54279987c9e0ed2c3e11c", "dsid_944f4416637044c9903ca2f0aa6d3395", "dsid_5055accef7204725ba113057da48063a", "dsid_cdae26161bd6419ea59fdfc5d93162e5" ] }, { "question_id": "qst_0101::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f19474cba1ba418ebd2d0aa5fc68c403", "dsid_cac385ca285e40c6b32d2fcf30f82b5c", "dsid_11907800370d48f2afd31607f3dfcd96", "dsid_c4b7fd8151c94951bca269fee10d454d", "dsid_5b2a860fc29047b4b4f68d0862199bf6", "dsid_871327086e07408fa4293196086d2620", "dsid_7d49ac2178ae4039ad8625a6c61e0dea", "dsid_8f31f8673f14440e96e3d948f74ab63f", "dsid_de7003ae573943b0b7b63c0b60f16fa0", "dsid_d9147b6f83154ef38f4816311a94d63f" ] }, { "question_id": "qst_0079::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9776bd75ffbb4ca7b10669be1512daa3", "dsid_f73899f0c0af4d3f80270910a4d820bb", "dsid_85fd61b156154b4b921cc6440a68cf2b", "dsid_3c82bc883aaa410c9751a6abf0ac53ee", "dsid_ef1bdb72ae104ce3806e713802377f6b", "dsid_94981f0fdf8448e2bd45ef15ad923597", "dsid_a0eaeb0778ee43c09a1a87a0ca0d53c9", "dsid_5826306ad74842578a6b217f732361ad", "dsid_26bac82420af477fb8be48193b95344f", "dsid_41ae740164754c88b36a015e63ee42a9" ] }, { "question_id": "qst_0426::conflicting_info", "question_type": "conflicting_info", "rank": 5, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.2, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3199f6a2210448788a4af18c72b37c3d", "dsid_540607d488c4447baa8f6f1408ec42bb", "dsid_a6a4d42e97ce47aa9509e779f67a6463", "dsid_2bb75fde3be24cfcbb35ecb1c14dab21", "dsid_c90bd544f3b5437fba711492db31aeea", "dsid_553e1651575a416dae6b92d019d2f9f4", "dsid_e17c81ef98fa49fbbf3fa99a263801a8", "dsid_03ea4f36d8f642bf96ff2579b6e1112e", "dsid_dcf32bf1f55d47268f646820a1b1ca90", "dsid_edba7c82fce741b4bf4bb61f5bafe412" ] }, { "question_id": "qst_0327::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_69e745b76aa34eab8721ba0c88482ec0", "dsid_b09e8509b3444d02828baedab1c84666", "dsid_3437967651da43819111a30e7bed89a1", "dsid_0a6fd9644aa24ea7a2e62b86d712c4f8", "dsid_c52c609c0cdf4211a7e8c57bb1da72e4", "dsid_44eb49c0cbdf4847b7bb7c80be8688f2", "dsid_389bf7ec1bf54db5913767be3eeb537f", "dsid_ebf57e8846c14360a7bc803076cbd03b", "dsid_c3f5e058c55c4cbb8ae27bf304598af0", "dsid_e16fd013c88049b0b85b27e5dbeb1393" ] }, { "question_id": "qst_0461::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_22b0987227cd4fdb94d67e5a62bbde40", "dsid_0fa69b4e576f417ab759af8e33cb049f", "dsid_3649becd3bd14f4f8bb788601f02fe46", "dsid_a67adaad749f4570a28508abf05d8a5f", "dsid_c5288dd4874345adb80f4a71c9a18773", "dsid_d27fd764d6084c57ac61e65f4ef5e600", "dsid_27e2296110434d8b997158a1f2b217db", "dsid_6ae2bb84d2cd44bd938bfd39988118fe", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_c54fda6423f24bd7996f2c79baa56cb0" ] }, { "question_id": "qst_0022::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_20f19570273148f28305253a62c95fc9", "dsid_a89681963cca406d83fda47552264c6f", "dsid_b131746ce6a3402883132fddbdb1e88b", "dsid_4697453bca0b4734bc3bb6c5e2204126", "dsid_673d0e4e642e4b13bd6b6c1464b50462", "dsid_59ccf7ad27ff42fc9d3dcfa619985fea", "dsid_9b8c88fa328242a28239866dc990cb7a", "dsid_53098bda08dd4ebe9875d92f6f18aee1", "dsid_6627d1d62b60478197902970db4b3fa7", "dsid_0f1e6ba98bc8489e953a2e95ba6e368d" ] }, { "question_id": "qst_0103::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_e88dd0c5a63f4fe8bd73783ce44a05f8", "dsid_911c14a0c2f54a27b9d7474d73c82aac", "dsid_248aab7eb03d4a3c9c581f07a573b6d9", "dsid_f86330ecd067401c822845816231f124", "dsid_5a4540141c924a87bca649878a051c49", "dsid_27cf84eb34214fce8304d542bb11228b", "dsid_d09a29e40cb44ba0a7e98f544ddec785", "dsid_50134686028e4b73bce2a121571f5e99", "dsid_91249de330b34aad8423c8e7b60e37a6", "dsid_16fee312c4c64e288373bb3cce935fe8" ] }, { "question_id": "qst_0234::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b16eb801dbc741558b841c7c3047e98d", "dsid_5bdcc429a23f437dae750c60f644c1dd", "dsid_0f1d57b723934e6db1be9f2e9a1398de", "dsid_8eba3a129f7845da9d7bd22a3a9f0f1c", "dsid_b68bbbeacd6347c2b4f231197f11ee58", "dsid_e5e77b671ea24182ad04a73a29816b63", "dsid_512c80bd4c654348b294c8a63c2f0c9c", "dsid_3109dbee251a467d9e5f7e3303b7aa0d", "dsid_4ecc265a59c5499da505794b95399398", "dsid_c2c983ffac43449a9d663472f0438484" ] }, { "question_id": "qst_0161::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3b35497fd1e448d2940dd075c1d68bc3", "dsid_2fb08b7571444bdf894607549642498f", "dsid_726bb419a3d84d72b73fd163ecec8036", "dsid_8f8d7ac4321e4d38aacaed9f44cac1ad", "dsid_0c95e3baeefd468ea1877da5363213cb", "dsid_bd18e7244b064950ab6542ac457b5b66", "dsid_5888e4cbab624cf3b21975485b81a535", "dsid_8fd6795d92104dd3998832a69b8df637", "dsid_06504b99e7a24721acd37486d8e41d3c", "dsid_c896f88ea22041caa48245eb765cf988" ] }, { "question_id": "qst_0069::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c2870779c758424eb1af2fe2aa3038ec", "dsid_14454bb8fa40436d847bb0af6f30fdec", "dsid_d873f1adbe654259a48b9ba497e7e4ef", "dsid_d74e611409784ae898c11068cd02cd55", "dsid_de9c1d389e964346a5d863550ba817a2", "dsid_8e9246ce32134104a0c16027f897a16d", "dsid_a9b90649fe6e4d2980cae3e3e87a135f", "dsid_d0660866e36b4877b1f828b395181c9c", "dsid_3237666f5dd74914b1bb5cc5dd7630b2", "dsid_dd5f0e9d5435414791786f0ab4cd0363" ] }, { "question_id": "qst_0145::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_182c34ee62c54118b48c1390e77cd3ba", "dsid_9243312efd444335afdfaa0aabf5dd89", "dsid_ec22e020a7c342fea2b5b28291546c6d", "dsid_5e90545c721e4848ab080343ec187358", "dsid_fc2eacd0eb224900a543bb04ae7cf21a", "dsid_f1ceaad5f8c74d578f5f926ea3c3cc63", "dsid_bcf25bccb9c24eb7991446b7fa696c36", "dsid_25d772e911ce425094f28f022f314d22", "dsid_e47c10f5903d46b2a1cd18a55b13f205", "dsid_d656c22e43034e2aaa89072fe1efa4b0" ] }, { "question_id": "qst_0439::completeness", "question_type": "completeness", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2cfe3f56de90412599f3c7fb1b1df829", "dsid_6c9ffcf5dfd34fcea647ec17fb7a591d", "dsid_3b7139962b28486aaa3a498cb375b2a9", "dsid_88bc012d9dc74b4f9947f88c184bd71d", "dsid_045f0fda65374286be9a414c3fd157ce", "dsid_6483467a98ed4097b79676e8cf447f13", "dsid_6d87679242a842b08571001f0b3d2dc1", "dsid_8d990a0d64f94f8ca7e7421bc2d5549a", "dsid_d9381f8b97e24d8f83f3c2ef32d21a65", "dsid_12b40ee174cd4ba2abb680c627e94bbe" ] }, { "question_id": "qst_0027::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_e4b8a2a6b6594d77b1880327d506619d", "dsid_13b17d1c09014c60a7fba5510ff0de37", "dsid_f309b7f44b344bb9ac66d5e552273292", "dsid_586ed806c1944b69b12f58d20e254b8d", "dsid_6d3921f9d2444e76b93a3ff9ffd81984", "dsid_dff824f5d9fa409c965fbefb1e848123", "dsid_4a83331fa3d540c48e96d913efe857d0", "dsid_26e09d34c7544bbdba9533c008016142", "dsid_c1aa31b0c6c34645955d7f8bdd622705", "dsid_1ce2d7a1ddd749f7954bc2c8362200ea" ] }, { "question_id": "qst_0418::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b837e1140d1a4c3d8efd2f42d9b6a799", "dsid_aff78a10eb974cfb99cd820f9ac672a6", "dsid_b01c12d7b75240c6a712313d764bd8a7", "dsid_294d72f1c7f54fc1bd8f54b542d56c5d", "dsid_6ddbb46c266647f4922b8107fd6ea395", "dsid_981f4a0281054f9aaaec7887156eebc4", "dsid_8678af84bf4d4f4f8e796ac2208185d7", "dsid_26fe354d38864e5ca426dabec101045a", "dsid_54236bfe8948415d8424eaa1db57f88d", "dsid_c0ecd55042c8494a86c42d8b8e02ab60" ] }, { "question_id": "qst_0024::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_74ebb73359364bc791707a92a3227e3a", "dsid_cf06e75be4f94156ac338f62e3ee028b", "dsid_e527ac7a6ee045ac8b6066677c7f93f5", "dsid_bd6ceccb8e974b169a1e56808d99504a", "dsid_a1345dc6b9a94cfb8e158017683207fe", "dsid_e47b4066ba4348f8a5f36ff9e123e492", "dsid_e70afd2743e1460289bc278c150b61dd", "dsid_90c8aa47b95e4a15855e353dfd4b4fab", "dsid_558b3331c26c4f99ab4af3fcfff4f9e5", "dsid_712a231c1dc448809f7efdf19b29e513" ] }, { "question_id": "qst_0339::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 5, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.2, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2c7df9209ebe47ba982b9382e102d292", "dsid_9cd367b225214f48a77dc58684edf6c3", "dsid_911dbe96d7ae41a2bf297fe997c71d54", "dsid_51219e8733564137a709b7a5642f960b", "dsid_81208556437644a0a4dedf168baf81b5", "dsid_14fe6bfd17ad4da8a1b22c4aa3bf3cfd", "dsid_1d0fc00f440544578ba3d6100f53363e", "dsid_07460d87fb8f4d37861a07fd1dfe0bb5", "dsid_57e15e403bc746c2aab6ebfb67f639d7", "dsid_82720922468745d283cd921b6d92a2ce" ] }, { "question_id": "qst_0285::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_1e7907ad4e834c389472f1f60af636c5", "dsid_2271e94991224ce9bb32ecc6766e84da", "dsid_d6ed5dd197bc4198b6a044c30728217f", "dsid_d41cc4970fda4b699fb8754e0cf0843b", "dsid_ce92c6e106f5421fba686b54851f7b19", "dsid_ba84e8d17a994d19a8fa9b37b91abcea", "dsid_f696b836e17141e19d3447ec101e13b0", "dsid_a3ebe69f7f424a6a96a15c7b7b1b31b5", "dsid_2832c51ab483482cb8a2594fb1a460e8", "dsid_7b91d35bb9cf4f85bc3550a14127980d" ] }, { "question_id": "qst_0091::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_da1a96a29af44f19ae6fae65f4a87ec6", "dsid_9eda2ec9cbda4188936246d888df1d0b", "dsid_0d359b15d28a4b87b7d371b30091732d", "dsid_a3481dcfc3554616a0b252b5fd6411bf", "dsid_610d2c74a60441e886fcc39e351a4176", "dsid_8d068725edf047268c39e1bbda95084f", "dsid_48eeb6667dc24b10b3d6368c5b53a8eb", "dsid_0dacf71c5c6f4b2eb3a7ed262773799c", "dsid_e47b4066ba4348f8a5f36ff9e123e492", "dsid_e302a5157bab4800ab5799672b2ae4d6" ] }, { "question_id": "qst_0075::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8c60fea2755245de913a78b2e29c617e", "dsid_72c88c17210d437e8d0db838d4f690a0", "dsid_11ec526dc14943b4b47b591dc9f47b28", "dsid_2e25db9a01874562bcd8b820c513c089", "dsid_cf2f6a4920a64e118550c99d7e297bac", "dsid_0eec015099f542ba949ada56d34453a2", "dsid_36b163d43c51450fae48cb0795d7cf22", "dsid_5ea82211a864441c8f847c690182d761", "dsid_69d842baf35a42538bf3d26a4080fadb", "dsid_a0de5e8efac04b8cb8b22b56ec11bad2" ] }, { "question_id": "qst_0470::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_aea4790d4bcc45859208d7d705f682a1", "dsid_27fe1b2619a44143bbf15407aed92961", "dsid_751f6fb0aa5a4d1dacd6ab613b3a5aec", "dsid_5c0c15496b5a4a51ab93fb9c853b6748", "dsid_5fa24d05c02045afb2aac4e64e65640e", "dsid_eef6a2e56beb486fb0c0436372eeeec1", "dsid_4e31660abf8f4a2ebf2c29cb721e453f", "dsid_e165a26030b64440a1d6eb3c647436e6", "dsid_881fbfffb0744f2cbf371f65b3f14245", "dsid_0128dcc681174d3f8406dd630c53c7ea" ] }, { "question_id": "qst_0084::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_06668ceee82747f29c1fe5220f059d6a", "dsid_3fb904923c5c434eaa2473ebbefeb16d", "dsid_ea23b1139d4141c483f46443089031e0", "dsid_dd20fbf1be174b9f8498dbb81ac84611", "dsid_cd47d0825b104d0c9b9e06f98a5f4d9c", "dsid_2385833d52ef4ca89c37f88dce247bba", "dsid_1e450059aa534ee0b1052c82767f7dfe", "dsid_f2f77dfb5bfc48e2840f8e023be3aba8", "dsid_0c0fa8b46ebb4223bb4511676c201832", "dsid_6011743a83dd47a7ad754f87cd587b01" ] }, { "question_id": "qst_0043::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_795498b20e9c4d38883c8c46ab3336b4", "dsid_a0de5e8efac04b8cb8b22b56ec11bad2", "dsid_e7e0bb32eb8b44b8bd6bb8a4e9071d08", "dsid_aabaaddfdbf040c4ab5534b1158edbdc", "dsid_eda10a8a384647768f1a30a51e7c81c8", "dsid_67bea99d2c424fc5b573aad68fca2c7a", "dsid_33ccd055006a4937b0ce9b375f3c87a3", "dsid_e4e5f7ec247b43efb0a19713cdc32c3b", "dsid_13811764ba9f4cb792f4fab84fbfc546", "dsid_2c0b893a4a704348ad6ebfe0f0ccf987" ] }, { "question_id": "qst_0422::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b8a286c123f44464b86cb4b1faa8dbda", "dsid_f2b5d64e164a4bb8952c2d71efaed8ee", "dsid_c5f0f39c0d2e430680b1b12b523493a4", "dsid_8baa852c57644c989b430c2f749b0cab", "dsid_4a2476f041294cd6a9fc0344db20a553", "dsid_d417b614467040e0b9f40ab37dece165", "dsid_8e1cafa0aceb4204b623394520a91b2c", "dsid_73c16e3b43a348c7bbd440c86a40e5ee", "dsid_caaa763133944643b120dd51469feab2", "dsid_dea9de3e5e904e4aa4e173e254e785c7" ] }, { "question_id": "qst_0094::metadata", "question_type": "metadata", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_cda0c4db44f44698b16efc093201273f", "dsid_b2aedf3ac1934dad9bd4d612b22b58e3", "dsid_f8ce92dd6d4941dc9d8f8f24e3dd9d45", "dsid_517537a232a64533afc446786d7b1ebe", "dsid_746c88d9520549c9b142985f3af6974a", "dsid_adf8269eee154eecb1c015c66942609b", "dsid_c529873f80de446aa2b972100f0c6d6c", "dsid_f7a0934d387d4b129077a8515e60801c", "dsid_0f65f6c989dd44798aff739ac683b544", "dsid_aacd32b93d4948ef815dcb85d605fb09" ] }, { "question_id": "qst_0081::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7a7c04eebb914edab49f734c8e814beb", "dsid_44cd54899c3d4d9596e345a40ece6b63", "dsid_62db09372a7f40578cdcb67962a9cf73", "dsid_71dac8bd66b54195b0b00e658d3dfca3", "dsid_3416dceadf934be791623630c5f809b1", "dsid_ae1faca1823d48fd8bb79f10692c3877", "dsid_aba4e56497e340fa9775ca0b67377c62", "dsid_09096ac3d6014989a8c8b54e4db58043", "dsid_21e9132e17914ad68ab9f4541d2323fd", "dsid_3a0b21ec74924d81897740b3e8729e13" ] }, { "question_id": "qst_0068::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_5e1fc88fecea47c7a2a74bc1ef063bad", "dsid_629a622594a24e4ca21e89ac82df06c2", "dsid_a3ff6f6eec4b44f69c0e88303c264c65", "dsid_39520fd8f87b4078bc12f3d57a53e362", "dsid_d125f64e37c14a60b68ebbdd81844638", "dsid_0592c49729a44b3e944522c6e98854aa", "dsid_58a34b025e1c4ccf8c97dbc9e042548b", "dsid_2b7b1f23d05c47508ba435dce0558072", "dsid_71e27fa4eb0a4c90a4df75a404e905bf", "dsid_45b99ac2827a42c1ad6f01d17b6944e5" ] }, { "question_id": "qst_0107::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_509b34ee9a334f68a693e61dfbf558c4", "dsid_566f5beae5304a76b92f2bd850a5577a", "dsid_342b24e0e85f4171b8689680e455c21a", "dsid_c31284f73f7947fcb0ff58196abe90f8", "dsid_3e2f413f5c424a2a80fd1cc48fa302cf", "dsid_81bdb0fd1e1b4604a5f43d6b6a658c52", "dsid_43ca3f055d7a4ec1ba4bdc5e57edc0be", "dsid_3aba1450367e4c2298b4e2ad5c0b6b2d", "dsid_559fee3561294637a1b1dd866910f7cb", "dsid_74cfb156c0e94229a8472370ee6eea4a" ] }, { "question_id": "qst_0079::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_85dd82f6a51a46c8a996388ae2762dd1", "dsid_9bc6725d29dc426cb2b261167fd567c3", "dsid_f63e4d18ec674e96ba30139d0aa7c50d", "dsid_310e387ea02844a589d5b79d00141d74", "dsid_a66727beace24a5baf22940b496705d0", "dsid_ad64590bc0c049a3bc525680bd367f36", "dsid_eb9be5671f2d4d798976c0577663f8c0", "dsid_07b08618860c4098b02f141651febf50", "dsid_4969bd1ad4bb459481463d85be465251", "dsid_14f0bf709c4e448b95473dfe02c9bea4" ] }, { "question_id": "qst_0076::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b097ab5d9e4c41609f47432dfcb1508b", "dsid_b3a664d939dd4aa4802ac81cdf9896b6", "dsid_8e85c02011ff4ddc92ff86bcfa3cb72d", "dsid_bebef79549ca4c2792bb73bd30e686dc", "dsid_98adce48b4074afdb2425dab5ffeab26", "dsid_4c400c2f6b6d431298528b4459edff6a", "dsid_2f0906371f294dceb7a034830a4ceb36", "dsid_c5b8c516b9a7423bb2db89bac98004ac", "dsid_06cedfb4b55448ecb275c78115120670", "dsid_19d5d2f6a4cc40cfaa6086ca39b2b741" ] }, { "question_id": "qst_0413::conflicting_info", "question_type": "conflicting_info", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6df52fdb96ae4edcb76464738bca3340", "dsid_9402f657ba454475b2816b3ad6995516", "dsid_fe3b67365877461d82634b38cae7601e", "dsid_9e5644aaa8fb4c34ad434aeb0f333fe8", "dsid_94981f0fdf8448e2bd45ef15ad923597", "dsid_845172d9dfc84fb1ac13457ca486f673", "dsid_513bf29a04584b6aba3f0c9084ec653f", "dsid_66a27222396249d094e87fd1c7533345", "dsid_ce564a4a8f6c45f7b547a7f7095af4cb", "dsid_b1e3c35ea53d49d3b7f01779d17d51c4" ] }, { "question_id": "qst_0363::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_b993fa19b3d44382b709e3107ab1ba77", "dsid_a820a561907547688422879d2dab3e37", "dsid_3ccd40af8e7c4d9da43b65fdf84eb6ae", "dsid_9a2f440e8f7e4801b76233767fcef945", "dsid_f81494a2de6f41a6a36110a07db697b2", "dsid_f2d5557e5a66420ab69617b62d95bc79", "dsid_44b11e967a5f4029aebb720891f66450", "dsid_130e02b5f9aa4ac78be9f0bb9b648f72", "dsid_abd6d7ab82104d61a354d7bf9a8f3b67", "dsid_164e247cc4874908a685434edd428979" ] }, { "question_id": "qst_0149::basic", "question_type": "basic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_afa8ec1f8a37490280fc39964fb75284", "dsid_e4e0e28a98b648038443103b0cfeb2e3", "dsid_178ff503c6fc4a4db593eace57e6938a", "dsid_42c3e2d70229415d82cac2f9b795bf1d", "dsid_d3825457e10041d5bdff57cb8eea5085", "dsid_c5adc254fb7c4833b966d6775545d3c8", "dsid_17962ee9e6e441d6a880332c89c1f1c2", "dsid_b05e1f4de81849d0ab742a4993c33fc8", "dsid_39606fc7f6bd4b98aa9821b31d057bbf", "dsid_3b9593d99be14fd9a5b3ea3d5cad22e6" ] }, { "question_id": "qst_0132::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d8a18c9f03de414faeacfda3ad53e315", "dsid_efefc1f7e55645808410b3338a3d5200", "dsid_854a9ca8daf54716b75c22377df89654", "dsid_5f16fc64309c4a2680840e3f6991b327", "dsid_02e9beaec3dc4f579c3dc8c892a8b05a", "dsid_863667bc41cf499bb5b430eaaf41693e", "dsid_b11d65828e2c4f379452c7698c8c28d5", "dsid_c40b65a75c4c46878b68994d5c68bfe2", "dsid_bf15bb68f59e4ec48ddb51084eb9e10d", "dsid_87892aad3aba4913aabfd2a12a1ebdee" ] }, { "question_id": "qst_0396::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f90b68cdda834c6fb4c76508d359c72f", "dsid_f940e7c95688406cb497b0b1dab44e99", "dsid_26db0d269ce04474afdae8bf7265e1ca", "dsid_61d15669b5684411b1c6df9e4d515b05", "dsid_00f927a59bb142b897f2f1d200325a89", "dsid_c3b14485712a448d8227f6a1efb3cbb7", "dsid_58f2cc3e008a4060bf5f9fa2677cb20f", "dsid_a5cc666385c242f2a37a57206cddace2", "dsid_f0b14f9b0cf2409baf9ee85b61b507c7", "dsid_d57c1953e54748f9802ee76d98a410a7" ] }, { "question_id": "qst_0435::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ce60931a1aea428b8186228ee40bb497", "dsid_a134a3fd3c244a52a8d3e1ff515fdf58", "dsid_431b0713c42847bbac15f3354776db68", "dsid_03e22d519f9b40a5ac61a03e5c4620d3", "dsid_8116cb3e580543d7bde1c57df351a3f3", "dsid_7fc490f8522b486c8f4b665d5189066c", "dsid_1e38da139cc648e387d299f627f2dad6", "dsid_76ce1fdd2b4d40d185ea0b5b27ae9d78", "dsid_b74460c600c24e9890460374aa27622d", "dsid_b7a54411963c4427bc8c72504b36a59a" ] }, { "question_id": "qst_0082::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b538d114ffc44503bc41c5ccf49f8fd5", "dsid_a041b647673a4de7b203e043cdc2dd9b", "dsid_ee3061c62b8c452ba359b1da6038b871", "dsid_f19e28a681d64c7da20a248bb0b5f960", "dsid_da0ca8d5d0bf4951bab18eb805d49215", "dsid_f0308bde052448f48d9dbc37d163c983", "dsid_8fc39be39e604a58834e73f5432c9110", "dsid_0257bdbb401e4b0a80aff76f57ca7ac7", "dsid_e630006e7945458ea50b048c24a744bc", "dsid_d34e81facc8f42afac1b43087a41ecdd" ] }, { "question_id": "qst_0083::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b105603f46224f539cb8a1b93c9aed21", "dsid_3b9483f629fd423d81ac49a7efff89c9", "dsid_946e4cd90985424aaf4a7536571cdaa9", "dsid_c3e9f8e3807d4b30ab782efbcc2e45d0", "dsid_a690b86aeaa84867be2190c36b1da63e", "dsid_f94eddd70c7e4a028e16c760809e9333", "dsid_8a4fc00a0d7e4957b2682914f9c21cd2", "dsid_83259ffa83a546a3adbffce5381833af", "dsid_7de214c03c674ce1bab60fd9cd74dabf", "dsid_d171e3e2d2b64dc686f4558085947647" ] }, { "question_id": "qst_0037::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c1a652436b2e4c949f5de9f5da227cd7", "dsid_e0c3901033834220b87974050eb7daea", "dsid_945ad52b9ae04034831d4da0e9d091bc", "dsid_3532bf9d2d334253ac90846ced71bf1a", "dsid_f03dc11c3c7947adada937e4e3a63e99", "dsid_3c79ace6733f476bb54d1d289894d09c", "dsid_a481dfe3eb264674b331b478a9793ef6", "dsid_06553e3df89f4e36bc6383ef75eb1619", "dsid_e9ed987738a34badbfc189ac5c8eec60", "dsid_a7ba10ff883b44b49f0bed909961e981" ] }, { "question_id": "qst_0241::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_586b30ff5b3045948083a2b9d2802403", "dsid_dd5a1af182ea4dd38ec5e9de375d9904", "dsid_af0820a88cfb4b9fb8881e72cdadda5d", "dsid_c47682a5eada448692c8dfb418c9cbeb", "dsid_d5793f5ccf964c8a808001a2d1a26413", "dsid_5e9f6ee1760a48ab87f5b6d7f3b3d42f", "dsid_200c0d311412478dab2d58fc1957a761", "dsid_745f0dc98bc741ebba4e17d0a52c57ab", "dsid_22a2141e89ce4565abd22fe7d5cb2103", "dsid_3e6b06f54c7f4627b3758acf62d00b1b" ] }, { "question_id": "qst_0055::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d7aeb7de2b6c4ae88f46f4b0e4f2aacb", "dsid_d34dbe143a1e4af4acf8cd6624ff01b8", "dsid_3d589feb625e402e96b9b17e8fa77c27", "dsid_de7e1028e7ff4514a626deb269f34909", "dsid_b9955b087bde4027825bdb129896ab67", "dsid_ba6c192eb8f14f4699cf00d85462a8aa", "dsid_2e3e9c317bdd4811a56e079b1dabeb8f", "dsid_1e9f3309e6904a96a0a7a031336a91b7", "dsid_e6c07b93a214446eb56318498226064e", "dsid_919fb882858145bca248ae15e8191371" ] }, { "question_id": "qst_0282::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_045f0fda65374286be9a414c3fd157ce", "dsid_d84bdbf7547f4feab122ed2f99e23782", "dsid_e85343285b9543d18c46460dab7fa574", "dsid_4e016427a93c49ffadd31fcdc1dca792", "dsid_a52a2354f2ef4a26be3e96dc2f9d6a63", "dsid_2cfe3f56de90412599f3c7fb1b1df829", "dsid_6afdc653d816440b821389f73841a3bd", "dsid_6cc25761991248b2b681ccef319a7588", "dsid_bdc205d1541e4e61a7a4b2f70c3f5583", "dsid_8129c0494af1479781d401363efc4526" ] }, { "question_id": "qst_0065::metadata", "question_type": "metadata", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_433a3aa8ed564e0f950f5982db86200f", "dsid_2052ba001287421486837cd036551974", "dsid_a96a28068bcb46d88e05ffd71a836b28", "dsid_09c9c54115d943eda12888709ed23695", "dsid_68ff6381066841fbbf580244355a4663", "dsid_7e82d2ac51114ca2b1c33ce55a96e499", "dsid_c4c26736e65f497885dbafb2f27596ea", "dsid_09729589656b4839b1c13f8d354145f3", "dsid_15099781d9cb49d79e82a8264ec15d95", "dsid_abe7a4bcfe2147eca3a3afc5287c3949" ] }, { "question_id": "qst_0036::basic", "question_type": "basic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a195b8bdde7a4f49b462b1ad2d2bdfc9", "dsid_3e2b9934184e4cb99c388c50f90d7e36", "dsid_c239203a0c7d433f8079475beb34b371", "dsid_b8cf75b6e0c748e5b9ae0fa3eeeb8c32", "dsid_3301f27e480a43f587e0601093e63e1e", "dsid_f94b93a4bc7948ddab45c371187dc6c6", "dsid_75ba67a1dec54308a05b9ca0a97dc8ba", "dsid_c002efa1e4c24409a7cd65c9154da3a3", "dsid_6b53af8a237b4979a1092134eb15c428", "dsid_e2bfd0c716374c88ae9e4af6040d0f28" ] }, { "question_id": "qst_0297::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ef4d0f403797422ab9de5e589431921c", "dsid_5367ff2e8f674d2798ea137a5e060f8b", "dsid_7c36f49e191e42f3ad2bf450391c2eac", "dsid_a150825e326648cba900845e8faf6ec0", "dsid_50ba230b8d194f51bfb724bb39a72404", "dsid_0f64a2fdc9664e40afc6792d8927297d", "dsid_8bc9cc9afc3648cc8963d1c72507a5ba", "dsid_c3190b4887674b2091951d372a4256ca", "dsid_d5aa0ebab2d9498d8e2c065b09dae62a", "dsid_daeab827c6f846578c1f7d0d1daa3f5e" ] }, { "question_id": "qst_0248::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_08a352047d7a43ca836dabb7dd50787c", "dsid_462f37885ea5406284727925ca5ded7d", "dsid_1835e03ef3c04dd1b3893ad2f951a7f1", "dsid_77b53da51e024b579e403d887f1e4dfc", "dsid_fae9cfc152664bc080de228d49cefce2", "dsid_7c52f59a57b9405a9fd26c83d43bd2f9", "dsid_2e9b2a8b364440ea99a6b2b15da49f5e", "dsid_f51fe807cdc749be8226d3867de81da9", "dsid_8bee5f2c68b849b1b58dcb8c6b1f5e1d", "dsid_30bd59f3c58a44ce928e4fa154d7aa3f" ] }, { "question_id": "qst_0156::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3c4d006398034c88a735eb9970ea0c2a", "dsid_32c5e7ca0e2e4eadac78b01a35cfed7a", "dsid_1aad84ede0d24496892a79c41c1d837f", "dsid_d78b80dce604425ebb87797c5835f01d", "dsid_1808fb06a2e74280b88d39d1693e6183", "dsid_cb0c96ce126545b29d5e81d1e84a8022", "dsid_daefe7f0da4e4f728930e1680241e3fd", "dsid_7aecae3ab9e74f59aa1c641774bf8c01", "dsid_2680a894a35b4117a2220bb308c4dc23", "dsid_3a31d592846e43ecb17d2bbc217a4cd9" ] }, { "question_id": "qst_0096::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8138b96e181940418cc5067b2630064e", "dsid_0f78717a028547e3839243f9c95f376d", "dsid_55d466da21434c6caf6b99680bc330a5", "dsid_8dbd4c431da748629df8c01dccd7215a", "dsid_a520e23435184b05b2e306cf3bc24161", "dsid_fd45848bbf9244dbbfbfcc83d1d08c48", "dsid_632f1e5f62244dd99cb6da9069b27a44", "dsid_bde7deaf58cf43c5990a278a6c04d2e9", "dsid_79ccb0c4801a495aae3a4f8004bd9a46", "dsid_bcd39906ef6840d8a88743fc74659024" ] }, { "question_id": "qst_0224::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_028f33218175446eb19771ca594ea33d", "dsid_25e59b29eab24930bfb23b5e14254ebf", "dsid_32a4350895a24e799ff59c82d4a37809", "dsid_2fd7a5a549f6438082fb15515bc0e7af", "dsid_34e200fdd069434f96328f53d671dc76", "dsid_8cb8f841aa894ac9a43664949237deff", "dsid_04c08ffee16f40179b3b30eb92070a6f", "dsid_bc29223c0d7a4b5dbac05ebf176ab6b9", "dsid_7573c2dc21fe469db741a857c0cee89c", "dsid_98584e12016e4eeabdae18dfead8d513" ] }, { "question_id": "qst_0066::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_8f5b801cebc34eb4b233bd0ef7c1b546", "dsid_a8d6dbb808cf4c5abe20e1c62bb99358", "dsid_546fa8f856944178a26b64437a128ff3", "dsid_273d7b2c1ff348c08f7d64b65953d1f6", "dsid_a842f958864a47d79c4e5581edd5d31a", "dsid_5a9b7fae28ff4eec90843beba457a94e", "dsid_8f183c6c6fe547e886d13caf1412e457", "dsid_f6837e26014041809d77f6a315e1e463", "dsid_eb768ecc1ff84a6f940ccad3b2b41170", "dsid_369c1d0ad2eb468d859d3d751645ab41" ] }, { "question_id": "qst_0042::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_27d90761ae6a478db1fcd6a29ffd572a", "dsid_96a1f33921374aa6b856dd3eecb95343", "dsid_0271fcc1742547b7a8f587085bd0f54e", "dsid_525c6cc0cc974b85a8ef9d79f89d063b", "dsid_515d7c1a1ead44de8d397487ba5824fc", "dsid_145351a29719460ea90ae229065bfa53", "dsid_64d412de139a4e70b9494cf75cb3346b", "dsid_8a27c1f488ec4980a787a5d51be0ebaf", "dsid_9d3d1641bd964c09af613801c922c69d", "dsid_6ee1566c1ca1482ba336c97cb4a49f15" ] }, { "question_id": "qst_0290::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b8fb9fd4b41c47aeaa6e009ddc179fa5", "dsid_1b7a5d6230f84eb0a38a97da0e0ce05c", "dsid_b9b715a0d07a48d292ac7124441ca27d", "dsid_dc6d425563f542ba8b91761aaacc23c8", "dsid_acded526a11f4ed8b211b9d5b6d4ad2f", "dsid_fc5e896d8eb4455fa9b186d04e0845f4", "dsid_a3dded417a234c7bb880ac061180da92", "dsid_d47e200a5734412d8126de7b3995390d", "dsid_e27812087bff40a7b716809128ada18b", "dsid_5f0faf87cc614ed8bcf5fc107af9ecfc" ] }, { "question_id": "qst_0042::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_eea41d353de049aa89be98b6f9cf5de1", "dsid_d5f20b1b886e4ba0b8919d97a92354c5", "dsid_8b16af25a6494ba69d730bcbc72d6841", "dsid_660e799acd934bb791c1418ff464221e", "dsid_770978f6e8f3414e88d67845f53532aa", "dsid_e887eca3501a47309ded7cd029e82d06", "dsid_8970c61c96c7447c979112fd786cf660", "dsid_9fe5b42cc2e045e6909e13192aee7f27", "dsid_3ac2751eb688477f9f73c86de89a7092", "dsid_786e758a14ca4efbac58f0d18a87384a" ] }, { "question_id": "qst_0458::miscellaneous", "question_type": "miscellaneous", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_9c7f8d6d2f2b4f0d9c2d3f6a1b8e4a21", "dsid_8cb8070d41604b67901e60da07713d92", "dsid_38a4af67314d46179ae4f3e7ff616661", "dsid_bb14e2f3925b4e27829c4f3c1aea3af1", "dsid_5e4346da53e8479f8588feb72040465b", "dsid_1adcde5022614df0a956436a0e534ab1", "dsid_ff0b820eecb84b03bc2fcd1fe1c14650", "dsid_6ae2bb84d2cd44bd938bfd39988118fe", "dsid_d583493514464fe1859fc348781ff251", "dsid_1e3a1fba0a914113ba6a5e831424888d" ] }, { "question_id": "qst_0242::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_192b9d8df8cc4f7b8a16b714b231ecfa", "dsid_726e843ced0548fe84d197c87ce30d52", "dsid_e1b59c3c34454b7fa74feb5a561cdb2b", "dsid_e10a550e74b644b88b40d98c0a35870e", "dsid_8f8894fd029842f59b113ea9054bcab8", "dsid_b00f5bf1db65426f9f5e765e4d38e6e6", "dsid_869376952a8d4eebacce7169f0bdee98", "dsid_9cb915beedaf4a2b913ceb1f1f5935c7", "dsid_12d14f86ccc9486e9062616b263cc14f", "dsid_056b0be16502479087e695dd8104225f" ] }, { "question_id": "qst_0347::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_b8691ba2e29542838afae93928ad816a", "dsid_a5ebe8fc667e472cadd7b9d914539434", "dsid_ea2d1b2252a0476fb8351fe49038ec39", "dsid_dab17ee5c3924a9998b17b3591bbae28", "dsid_ddcc9309157b44978465c4a6b8b4c9bc", "dsid_10147c3cc869473f8f1ed3a0ca04a221", "dsid_6626042e2e414e7785cd5e4f7fd96f48", "dsid_ecf32f68ff2049dcbaec9294337da274", "dsid_370aa37572db4327bebd334f1929514c", "dsid_2a6a3fb047a14608b73c57b017b27ffe" ] }, { "question_id": "qst_0200::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fd1fe7d21e8e4d04bc3f23695ca66563", "dsid_49f6e79392744fabbc2fff0454f6657d", "dsid_a781a28d47d64e4bac47af3a8ce3d889", "dsid_6b453449331c43dcbdc4718217520db7", "dsid_9fd9f39af28b422a99f1f483cca61c6b", "dsid_de3218684e914ce3a0c78fe3c187f9b6", "dsid_065f8fee2a4a440d9e298c51cfc6f618", "dsid_86f5fede8ee14dfc886c1e50a5b82fbe", "dsid_8755c1c726404aab9260c7323d5ce77e", "dsid_0438131bc31b4937bc3746135334eb65" ] }, { "question_id": "qst_0083::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0abbd8a3a39e4cfda70dcc403b22f64c", "dsid_920ebc43159c4d418cabaf4715ecb995", "dsid_dbbbc91e4f40473db342e94882784e1e", "dsid_195d8d92edbd49ed8e747ae8aa1478dc", "dsid_41a817d550b64abdbf9fac78ffe058c4", "dsid_ab09b94479c44887ad5874177ffac633", "dsid_f86330ecd067401c822845816231f124", "dsid_53a865ae4c844158b870c8d083158adc", "dsid_9bf978987316442fbae2b476c9613db9", "dsid_3e97de376e7a4c4d8d1e0b3cb8a80904" ] }, { "question_id": "qst_0141::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_acc33ff059404020b0ff26e19ad2b107", "dsid_0f669a940f374e6ba96cabc93d30c5e6", "dsid_dc2206d0b8a6433fa69a4a7ff79f6395", "dsid_db40a21312a14728a325a83f84ba10aa", "dsid_6cd910f09dbd4e8da4b66d17afb9c4e9", "dsid_cf9fb418fa934a44977938808dcc3ca4", "dsid_b9de16818c534d8c919da083a09613b1", "dsid_7f9ac3dc4c9e4d5eac3f4fbafef7bf72", "dsid_1fe8dc0642d34c13a35ad33ebccfbb41", "dsid_eeff65e7fd574f88b205aca02e152c69" ] }, { "question_id": "qst_0445::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.125, "retrieved_doc_ids": [ "dsid_cb0d800639e84d68ae035596c7f6f1f8", "dsid_c4421c194c57422e9f33d799711fd7cc", "dsid_e34331a85af34feab780648a00295b5a", "dsid_d4ca84e26feb40aca75ae839f2f06504", "dsid_b9bc36c1081a4dc1942108d3fa182477", "dsid_b329231826b04e2dbda9f12b8db13a93", "dsid_e076bd2c5dcd43e0b3b244457659ec6f", "dsid_c6ec6df4e6d34c68aebead677ba51548", "dsid_2ad75f27c36b492f847d5b9c63492e4b", "dsid_5c09bbc463f14812b9aac1252a07f4bc" ] }, { "question_id": "qst_0371::project_related", "question_type": "project_related", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 0.6666666666666666, "retrieved_doc_ids": [ "dsid_cc26932462f04aacb259a6bed02b100e", "dsid_7a17252f6044474186155115a7de5ff2", "dsid_fc356e32e9a64c3199be174cb91432eb", "dsid_1c0c708585a043668b6ea87074629769", "dsid_49c27f8f803541ea9f421d4c573c3d26", "dsid_bf60c824c78442a28792b411575974dd", "dsid_36d6e7b019084b16a2fa17397e5f00a7", "dsid_522181eff54c442bb95138cb02af5f0a", "dsid_a1d8ea3adc68431ebd90cc40a51b5627", "dsid_ab681d3d88d742acb47f7ffae37a39ee" ] }, { "question_id": "qst_0277::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c643d947a7e64547a6546ec57ec1afd9", "dsid_08ffe8a6dd8346d7885af25e51585b2d", "dsid_7b5a30e323f5400bbd3e00dc4fe9712e", "dsid_7c9995a71a504d1ab440f1bfb83377df", "dsid_c960c001e9f2479e9ff9567a2c1d795f", "dsid_9b02f194b0014c948be51a938c984753", "dsid_683dc2cf9bcb41b7be76949c6e0965e6", "dsid_ffa04a7d7c9b4c0b8f962d23d8c08c30", "dsid_d665425bed5d41bbac50f566f4497a4b", "dsid_0fa90b88a6284fb78840ad0c8b0cabad" ] }, { "question_id": "qst_0322::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_bab6aba4c06e41e9ae46b9355f644a7a", "dsid_dd34a1f6919e456ead57b0c3adbf4d18", "dsid_89cf9a5275164e75bb891679151e292e", "dsid_f62e1ce79fe94862b006e2df567ac5a7", "dsid_e2141c0a04894fc3b47b33f8e06adf5d", "dsid_872783d0ca63433aa0095dc4557dc3f3", "dsid_e4427606e1eb4a0d8192b8bff51d849d", "dsid_d20e25a1a30b43f1beabb99439944366", "dsid_6914d958fc0242639ead3e6a6c9a4b82", "dsid_14106824ac654eb3b84f5fea7d9fdf62" ] }, { "question_id": "qst_0109::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c9e70cb6eee1471baba213a976a8dd05", "dsid_f734e4e3dfd846c7823b2213263fa790", "dsid_2de9c2082e074140bef60a671fdc501c", "dsid_14f73462acc24ff09bb8bbfefb4c480e", "dsid_02878ceabbd745c3a0800381eec3a660", "dsid_3d4654db2ab947fcbb877b8d0afe6925", "dsid_07b14a162ea54333a4bf76e9e7e45f9e", "dsid_828c7f2710b0435c85cd004126aa209d", "dsid_92289c54d6b046c0b8495394f4951acb", "dsid_466d946096234b50a646d0c896e9ca00" ] }, { "question_id": "qst_0294::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f352f8454fd6403aa6b4931ae2733362", "dsid_772277ad60ca4357b321b8a173b99a3d", "dsid_2973d8c94a694a2da87673c284cb7c04", "dsid_88a4bc9c35ba46dbae03e660a8cebeba", "dsid_af9f593080734421b56db8dd98d9ed6e", "dsid_741e17f34f654ee7b1fdea7eef31af8e", "dsid_2e27ba3769084156b53d050a46e93fbf", "dsid_e7e0d27557d7441fa7c5fcc5dfc57b75", "dsid_1e1d450325c144efbd4c6ec7033d82d7", "dsid_ea340c97ba4f48d0aef6e76a7b11bc18" ] }, { "question_id": "qst_0284::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_6cc88029979041529b67a9c4c4c7df77", "dsid_f305c85fab224256a7330bdcd1536353", "dsid_8f631719ba14402abb361802f9cca4b8", "dsid_93742074ff2244e29d5beb445f74318f", "dsid_b6307d9d3c0f441e98ab07e034178a99", "dsid_8d058596be3847148bed3f5981f0d1ae", "dsid_4eca941be62c484680660935b66848f9", "dsid_5910b2d63fc9479198c6f77bec7a1e3d", "dsid_f790417e26374e5a99a3793cef8f6968", "dsid_1d7e3986d1444fc0860bc7e77d166183" ] }, { "question_id": "qst_0051::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a105216ad647439ab4d1d0635df2ecb3", "dsid_3b7f029c30c546d28752d38e0f05bdf5", "dsid_b7aa9260ff1b4eb586edf78ba2ed9ca9", "dsid_72b2786c94ef4143a947759ad7a071f4", "dsid_7aa18aaa818d4d8b98491fe541723548", "dsid_a0f85b6b382f4103b021b3a20bd22124", "dsid_578a4cddc1b2418a8b3011f6563dc214", "dsid_8856551b6a7242148b54b271e46463d5", "dsid_2f5b4312d3024ce6b3661baabead25c7", "dsid_f86b4e5a22ae4dd39e4efe9c8cd7cf52" ] }, { "question_id": "qst_0016::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_3f7f32874e9b4ba49ff4fe28c0ef5de0", "dsid_8ec77d56177644f88218faef2b7d8316", "dsid_3848df9a653f4b2da022ecb753587654", "dsid_18054493573f43aabcda10beb6d2be59", "dsid_f125690b111647cd899471b2257b51cc", "dsid_710d0dd0c5294415864b10986219da5a", "dsid_a2c21138d0a346f3829f79b8294fa762", "dsid_1b17615c99b74b68b50924e7bdd8bd26", "dsid_4289f3f1fb54444f81ad6fb1edc6dec3", "dsid_d6236d3b3d314545a39a6cb8503e8a2e" ] }, { "question_id": "qst_0259::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_fc009de77554459b89e6b4340e8eca82", "dsid_2dedc3d8d3684c30af553326438a0ff5", "dsid_6c09c40e9f404f9d89dc4251145307d1", "dsid_918a2ba9aaf447ea9d9e076fc7212af7", "dsid_6ce84394f2c74b0f8a5cce70d204d852", "dsid_67cc82eeb08b4b65a05d93e1919a144f", "dsid_42295edbd56e43c9894baa9d280fb64f", "dsid_6411d0a053384f6eac6bdb8356f9b7e1", "dsid_e7719f74ddfe4cedb809b9e826108930", "dsid_bd2a2e090a604950a5ed6226f82400e8" ] }, { "question_id": "qst_0203::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_83647b9fff63443896977370fbd384ac", "dsid_8a175e69fdf7472384c657448acc9472", "dsid_3407ac164f6744d4a2c241e0c5412b59", "dsid_c35ce749b81f4bd59537a72c0e26f587", "dsid_cc882f76effb41aeb2fe019d27bc3d91", "dsid_e69a1b81fbf94e0ea27797f2232530a7", "dsid_d34fce9ac3104a509da5248500ae03f1", "dsid_800c4280d8d74533ae12d75c77b1efb4", "dsid_b637cca914da4a50aedfbef2fed4bd86", "dsid_f6d58fc9be6e4a808ffd6822333333dc" ] }, { "question_id": "qst_0408::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_e190b0de0d27466fa91bfc18b0815311", "dsid_5f119369627543848ddc03d6805509dd", "dsid_e4142c27bb5447bc92a280a8bf1c36f9", "dsid_54a976f38c8940efbeaf82dba5b6ecd9", "dsid_4856d1d79a5a4e2d9bdf82f2d0e13ec1", "dsid_805f902eca9a492c8b1ea7a6b7ba7edf", "dsid_378ac277dfa744c8acd16b30af9e364e", "dsid_190d9732b59b411d8a6bcc59fd61b309", "dsid_5147667611a54584aa272d1469d8f6a7", "dsid_5e42acb61ecc4c5599ffccd619b87ea2" ] }, { "question_id": "qst_0126::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_dee213d420dc421dbe3cdfd9b96f4a37", "dsid_82ef491871f24ca9800f8060c76daeb7", "dsid_27745a33354c4f5089efc3eda3574794", "dsid_35ab7068d76e40dba9e3da3b7413ca70", "dsid_4835bb2555f54f62a2370e1c179fe95c", "dsid_310b703bb5b04dc7ba499dfab083c8e2", "dsid_22d7ec6499bf4eeda5ee8d1c3991be04", "dsid_7ddcd9c603e74e86abd1f736ee1d61b9", "dsid_be7635d0eb2247d3aeb05734ecaf5b42", "dsid_0e4bbac9aa8f4d4dbdb6810ea6567a7d" ] }, { "question_id": "qst_0271::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_db87c0dec8444b1d83b06cbbe725d3a8", "dsid_0a2e4368b95a49c4bf9e8da4ffa45c7e", "dsid_e405cd8a5f2447849fe200504b5a39d5", "dsid_7309ab9b5eb94ca7a3446f32e8bcface", "dsid_645cf10941d94bbda0d9fc6ac673e883", "dsid_5a185ba825aa4a68af428f5ad1aec7fa", "dsid_5da25063af26461994ebc6c0013f9432", "dsid_638bdb9278184f4f873b02c6e55af122", "dsid_9851dcd734bf45ccab6acea0b9b51f76", "dsid_c030f56acd5a4447a79be25683afdd87" ] }, { "question_id": "qst_0135::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_63a1c6c12e1e4d9cae67056673633192", "dsid_a959b5fcfa14486282145b8cde138f2c", "dsid_a3f4a0fad8d64c19bb1abb4bda52395c", "dsid_ad3961a5bbd846d09d101bfa951ab739", "dsid_a17932f8bf36453dbd5444a3df978708", "dsid_6c27b75e4af44d5fbcd22810ee6a3392", "dsid_3584510365fe433289504573d3c47e6b", "dsid_69921bbd7dc4415197062526429b76ae", "dsid_bbaf064b5fbf45e4a639c2e51830fe5c", "dsid_5b9c421b4c354a72b035169a6d9de2d5" ] }, { "question_id": "qst_0065::basic", "question_type": "basic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5e4fced5bea4475c83a105ac26aeb54b", "dsid_42d51fc5cb7746e4b1c9200136795a36", "dsid_383edc3ffaf04330934f79c88f57b0ac", "dsid_adba0fff3cf94faba1a64677d66aa19d", "dsid_a964ae854bb640ee98273730f629f787", "dsid_1eba87b0b11b4f31a006f0cf164b8509", "dsid_4b2c6c600488467ea8eaea64fd4e4aaf", "dsid_a270872f4f334eb0a0538379592eb522", "dsid_618649e367d24a1eb3bdcacc0a2d7da7", "dsid_964f838c97bc44feb56077e3d67ccc04" ] }, { "question_id": "qst_0050::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2bea8116212844eca26c9c90052b0ec2", "dsid_089917448e654911bd186650ed12feb9", "dsid_f1a3e4d11e274a22b0d5e646ad0ba9f8", "dsid_cd7be68e755f478ab32333eb90c8e5a4", "dsid_aaaf6ae3258d4fe2a6883f2cc58e7360", "dsid_09b15c8027114823b855e6d135df83f3", "dsid_be8d2802cf3245088a7e7f861f317b31", "dsid_11c048aa9480427384e13ab303e85512", "dsid_9626f1c17c854c3291f02e3b1d3db535", "dsid_4b955668c96f4ec3a119a33780f97d61" ] }, { "question_id": "qst_0157::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ccaf626687b940f3a7309e09c1ed5c1b", "dsid_a9152bd748f342999bea434d517122a6", "dsid_2132c5730307400bb971dc895c00d1e4", "dsid_0753a5dac4304fd4813d124865b80174", "dsid_3650c6099851415d8711e83c8cb6ad33", "dsid_189913bc1cc94a799a412fc024044c5f", "dsid_1afc3776799b4f0f8aa74df6b0282328", "dsid_c5d31f2acf3349ee910e5a63ce3f631b", "dsid_4c3ffe5a1d9b44bf8ea8460e42f5b286", "dsid_23b264ebc5ee47979bfce01157b2b2f5" ] }, { "question_id": "qst_0240::semantic", "question_type": "semantic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_4284b69805e24a8595054cbcb1398e59", "dsid_31059f73d2d2409488178447167534e6", "dsid_4f3e25d63e044e3091fc4ab0d9c5a138", "dsid_4fcf28da4953493d9cdc93cf86cc530b", "dsid_fc34f401b1724cac87b02b53c49b14e9", "dsid_5402831508734a51adf649603be4b075", "dsid_94098e0c64e14361866d21da3bbbd283", "dsid_9ec31084a81044eea561dd829b8b117f", "dsid_0e0a3b2a7377420c8360dd2a08ea8ba7", "dsid_490ce68e549d48fc869e881d9bc77d89" ] }, { "question_id": "qst_0094::basic", "question_type": "basic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_82e08a01e79b4b8aabd6a957097a0c56", "dsid_e4da9a6e464448cb94972d6259d476d8", "dsid_bec90d2205534ca9a016f244e13884e5", "dsid_b793195a94004991b5ce288b087c7b8c", "dsid_41666cea1d644dd7a401dd6d416c9fee", "dsid_ee3061c62b8c452ba359b1da6038b871", "dsid_a4c42dbf64a540b4bcac55013098b6cc", "dsid_7a684a1075274093b41284527f1cd13b", "dsid_1f8b9d86e07a4f639d487a74a7971ed5", "dsid_5016f308bc4f42f3b9f85c90e40c5b9c" ] }, { "question_id": "qst_0080::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ad3961a5bbd846d09d101bfa951ab739", "dsid_deae60bb40f445ec8b0416aaf0269253", "dsid_68d203c17f6d446095947b4788e8a083", "dsid_a3f4a0fad8d64c19bb1abb4bda52395c", "dsid_6a118a3fa8034c78b9c42303cc5440d8", "dsid_f619f7c8041e477aa46f6d7aca953528", "dsid_dcc7045dcc964ac8a58bb29b0270e2e9", "dsid_b135486896b5442bab43e66e6a4191c4", "dsid_682b716dd06649cfa62d2b5792650cc9", "dsid_4a6d655db9e3439da58cb2060dbda494" ] }, { "question_id": "qst_0404::constrained", "question_type": "constrained", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_f2f11e717baa4505bc6da6c8887aaa7d", "dsid_5b2aa3e1e8b54705875b9996322d762e", "dsid_b79c7fbba1884ff487bd2a4bb79b5598", "dsid_025f5bbe435748ae9c67ba8a45cf608c", "dsid_cd0a915e779b494d9a1327269746d51f", "dsid_4b31e74728154fd68a4739f5c6c8382b", "dsid_cfc5bc49b0d4481f986d0c058b790706", "dsid_2adcb851dc4346c5948de37ce881f150", "dsid_578b1072317f484ab7fc50b86b3be0e2", "dsid_e13f3074d35f44b7a3aaa697ec5fe125" ] }, { "question_id": "qst_0164::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_5c05d2a74c484906852925d1cf1daa70", "dsid_bb2b60e869d74dbc82c8f2ce60224e95", "dsid_86563f7a0b5a4852857400e46eac91d5", "dsid_5650ae3bff08466f92289cbc565c7d2b", "dsid_666f0e773c9c4a28af0f7cc210031215", "dsid_0bfbf30b70854702be9eb768f2fd7a62", "dsid_0a2cd37d53ff47d4aced289cd9a76fe8", "dsid_60b6b4cff8f440899bb213cbfb4e90ed", "dsid_8d97d64cfcb14cd3b13589b86b0a0c02", "dsid_50def288ce54494ba43a6e5b9efff182" ] }, { "question_id": "qst_0312::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_ed9adabfe1b4445c8a14d19ce265b97f", "dsid_0646657fd7e3448faf959a6883257e23", "dsid_c2a137fd3f114c268cad5d2c323af502", "dsid_a449c9fb7927461d8deba42ee39d1921", "dsid_933b17b27139483f9e02cebbad3a7b64", "dsid_95aaab5e5a6640ffaceadee4bcdb5f76", "dsid_635fd169c808444987f7de6af193e8eb", "dsid_13e85f84f35d45e8aa472cb085c7c1bf", "dsid_fec25e7f54c449f69149a71abe09bd44", "dsid_fadab57fce044302af798a476431d3b7" ] }, { "question_id": "qst_0215::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a56c6ba5a2094d8b83c6c70d025d940e", "dsid_e098932bcbac4db5bab53e5eba1c3dbe", "dsid_aa9506249505475999f9175cef0abbf7", "dsid_d7b6b915ba6a47d98fab7dddcc43e50c", "dsid_2463eb6e5daa46f7bc51be33788108d8", "dsid_3a2cd5b761ee48639842916ab3dad744", "dsid_5611de8404a0427a9764223842bc6a31", "dsid_8ed07df6c3b7411d94410c1584f67a7b", "dsid_f135a85feaf3453c93a0d78ebec1d457", "dsid_a16bcfd8386143f791046fa52b672669" ] }, { "question_id": "qst_0265::semantic", "question_type": "semantic", "rank": 6, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.16666666666666666, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_53d73ad387954e03a2b1b2970a939486", "dsid_3e9d1abb3e9041c297f6e392d7252265", "dsid_1dad3b2769e34d63a3f7f6a2741a2ed2", "dsid_033bacd65ffc48c59ec27881b70af6a2", "dsid_4866ef84859a43b89d77178c28d9513c", "dsid_876b1a31bcc7409ab560b9ccbe5a0d41", "dsid_bad6bd97d4db46a59cb15721b6537cd8", "dsid_4ecb42b192a1421885f6fbe8a7cbde8b", "dsid_b127efa7f884490fb4704c90cc8f0592", "dsid_986f0466ed5640a09902c5e71f091ded" ] }, { "question_id": "qst_0139::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c328881eb02845758bfa1709735a99c3", "dsid_4ca3a401d73e4469a53f47ec37affd6c", "dsid_6df5253728f64323a3c205d137026550", "dsid_14aacc9e1b234d6eb34ce88b04d5974b", "dsid_68d4086cb31a451a9b814f1d0c467c79", "dsid_b4ae48b9242e40a7a01a7f37ac0c072f", "dsid_edfe0e13a9384fd9887725e3c2688f2e", "dsid_9817cb9b20d94b6d9a35d6298e9ff6a9", "dsid_de47f102fc86470389ade3bcf7a1f39a", "dsid_065bffdd58f04eb9855398cda42f0fa8" ] }, { "question_id": "qst_0369::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.3333333333333333, "retrieved_doc_ids": [ "dsid_e7d8c585e8594fe481975515246d04f2", "dsid_31be9c3c708a4a5797d50a579c137587", "dsid_a560a94faf9c40ec9782d125d94fc009", "dsid_ab4664f39ea64fb48c7a924fb2ed3a4b", "dsid_5f67d44565c54a808ea851c9a049c2a8", "dsid_433eca62198e4fd5ac96675be3b7b6e7", "dsid_4ae647ae1e7a4ccd95be2bfb484796e2", "dsid_a360682237464cd196d9c439374d6520", "dsid_3413bb93a38549439b040f199687ce9c", "dsid_30a566647d334042987532c91410fc97" ] }, { "question_id": "qst_0403::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_a98ada14af2142a7a46266584d05c846", "dsid_9479faf893494f40be6f4593b656dd73", "dsid_c1b16f241ebf46caafdd845edd2c3da9", "dsid_2b547cc853804444a811242d53817f74", "dsid_400929f69f2b4fee8803b04f6aa50502", "dsid_242661c8aa1341148e493604f4733d4c", "dsid_b11d65828e2c4f379452c7698c8c28d5", "dsid_37ce7f83806247d3abbc9c111168d41a", "dsid_08d289e5283c4f17bc7bb2a19bbdf342", "dsid_8cd8888f9a1e4f9d9799b0331a59e620" ] }, { "question_id": "qst_0194::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6049efc70f294b48bff3cbf52ffeb128", "dsid_0c8ea3b2a7fd4392b1f480fe5f0c4725", "dsid_86d0a27f0f5e4a4c997e140b73f81150", "dsid_805483a9523f4f499d5cbdf72edeabbd", "dsid_0390dd5fb1764689891eb8489c035c8f", "dsid_99d1fbf95f9d45e699c4e4327319e6fc", "dsid_f03bf31eef8a4785b9e65ded4e13a89a", "dsid_e23bd5cdf6e54621a9ffc1766c41b51b", "dsid_8d33d34055924b56851c12534ddc67d3", "dsid_d41dc7d257a74f9fa973b17d408ae91b" ] }, { "question_id": "qst_0214::semantic", "question_type": "semantic", "rank": 5, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.2, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ecec45fdb060420ca809f6804e8e3b78", "dsid_ef708430fc714061acc3411b50df985c", "dsid_2779262caaae4790b48f421eefe2bcf2", "dsid_328b258e0f664bb5aa545595343dc2e0", "dsid_303196c831af4d1bb38a1e2546228a09", "dsid_d013ae8a1a2d443795696cbbb44cbc4c", "dsid_149604f2fd244a3ebb86ce6657039d10", "dsid_fb1ecd1679de4b318b0089c84b9b1cf7", "dsid_397ed85c900449dcaae969f6823a395a", "dsid_14ab5ed61abf46faad167f1c56e16abc" ] }, { "question_id": "qst_0018::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1f81f208f51a4aacaca226109bbedc72", "dsid_4648209b4d73412bb75e50dd65185e23", "dsid_df3e727ec0b5461e853466312e17aa96", "dsid_be6b63fe51e74bd894700b65030def0e", "dsid_61dca8c9612c408bb65c5c7d446d96fe", "dsid_dc754a26abdf4f4380b55420da36674d", "dsid_227fa7c5a0e8490aad48847d735bd196", "dsid_568e757465f44d1db10bf120f977a0c6", "dsid_9e164d0d18854df2bf884cf485d4f05b", "dsid_50639acb92d3478ba5f7361459be2c05" ] }, { "question_id": "qst_0175::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f02a338c2dd546b3b025b5d5d429ad4e", "dsid_0c26723cb88944f0a7bc08d7eacb4217", "dsid_6d555ff468544dc3972e3fc680748cd5", "dsid_14593afb16da4a0288e8ddbf417ac809", "dsid_a5f01bd635aa4596aa459e5f0bf208b5", "dsid_d190af74315c4a98a286a4bb2de8cacc", "dsid_ec24e5cdf7e04f44845db9b993ac079a", "dsid_15cdfc20c9e0467f8de28db0667adaaf", "dsid_b2b58499bd844ec3a41e287049559716", "dsid_49d6f426061e4701a97e3925bf3e4398" ] }, { "question_id": "qst_0160::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c84c8581594e47d49e82d024a6da915d", "dsid_409f56cd415b48b09fd0187ecdb48b1f", "dsid_4beb4aaeccdc4ddd989d4d6d23f4eba1", "dsid_b41e703be8a04b148db5f8e2237f67f9", "dsid_fdc3fb9502544d62b8c1e782d0e73bb3", "dsid_d17f040352fc4b78bde0e93c6c4a3e4d", "dsid_80c521b2658f4e969d712c472ef65383", "dsid_3c44e22f5e814eb38fbf3c1cfe29bdb3", "dsid_cfdd9ee880144f1fb338a5009c98b389", "dsid_5b98d7b923f34a9abd98e76ef940ac8b" ] }, { "question_id": "qst_0063::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_e555234794034e218991ca49a2ffb3ea", "dsid_c1f99c50101c486bba6926609637b94d", "dsid_31483eefda0b4b5eb4bbb673831da02f", "dsid_c2b48ec4a11d465b94744215b661b6eb", "dsid_715ad99c826a4e1fb5002c553f4b16d7", "dsid_c880448fba214acba0e50dddccc3242e", "dsid_562d40ac95ef4410919e6103788b9b6b", "dsid_6bc0b574f82b4e8c98b1cd5fa4dbe9a5", "dsid_06d163e4eb3848f0bcc496366c7bcf4a", "dsid_cf4056237f4747cab4a5bde4728ce77a" ] }, { "question_id": "qst_0062::basic", "question_type": "basic", "rank": 8, "hit_at_1": false, "hit_at_5": false, "hit_at_10": true, "mrr": 0.125, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_bf5f166671c042e98d2e41aedbf7987c", "dsid_b193c79da1894b548046571c89655fd0", "dsid_40aea29b6308417787932cdbbc478e1e", "dsid_a48726cd740f4ecc9479e38ff53649ab", "dsid_438eb5db87234854a830d0cdb638153b", "dsid_6edfb57d2f384f1eb2a0004d442b7800", "dsid_85203717c8834da6b946b329f3258f33", "dsid_702bc520c52040428799fed41c63c8ca", "dsid_ae3dcd1dd7ec4f37b1e795a4066c0be5", "dsid_2dbea63815614e35a7b98e86fa522bef" ] }, { "question_id": "qst_0096::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ca09c9594d0f4293b971c96fa9ed647c", "dsid_02dfda7a1b864bd0ac476e04815ecc94", "dsid_5c1c408fa554409fac6125db3ce9b387", "dsid_d98c7ace39c748f89e5328e9e7db9402", "dsid_86783f1b3d154ac7b57c6d3d71bb5efc", "dsid_380aac4dd4254d44a2b72d253fe0e268", "dsid_eaf55235a63e4f27ad39591ab97f6fac", "dsid_8d2187b4c5e144b4bdb07f13bea1eacf", "dsid_82d50eb5c3984502bb8b4a50fb9b4dc9", "dsid_d84bdbf7547f4feab122ed2f99e23782" ] }, { "question_id": "qst_0082::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_273d7b2c1ff348c08f7d64b65953d1f6", "dsid_dbeb3950ed074b20b8112e60e9ada150", "dsid_0014021f33f442d49de9cd34edeabe3d", "dsid_43b87c65f81b4e2ca03ef919c456effd", "dsid_9fad67220fcc4e6596b8f2cc292872d1", "dsid_b558dd44aa6141228f8cafc23dcbfe89", "dsid_f1dba37a605140399cdbec7160ea9c1d", "dsid_75fce8a278cb44ad9aec921acd2276ff", "dsid_3695ee3943904bd2911e71c604a9bc7c", "dsid_b88a967f51b246ca8ba256683205c683" ] }, { "question_id": "qst_0010::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_97846a3d292e4c778a0315c1b398b97a", "dsid_e16c1632730d46a289494796000c1a24", "dsid_7cecfeca064d4328a40440ded42fe3bb", "dsid_e470e44f1bf241a68fae044faac5418c", "dsid_5fa372ff5af94fdc93b9b5d0685af19e", "dsid_029120fd74ad4dbebaf8d1acb279a6d7", "dsid_bbf4506bc8d849e5a28a4c2acaefd3fc", "dsid_dc1807ac218043ddae77bbcc8a7356eb", "dsid_c5806a46769641af8474babb5e19f629", "dsid_895e58d9d1e14280a1a492c13ccb0b4e" ] }, { "question_id": "qst_0391::constrained", "question_type": "constrained", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_0355181b7fbd471f8d570bd63ca52b43", "dsid_0b0595a6d39247ada6238d8311b587f8", "dsid_164b94f1756a48d78badb0b0009489f4", "dsid_4f68b1fb107646b084510aa7ff9fc2a6", "dsid_594a81b3332945e7bf596875af810ab7", "dsid_ceae9e544dda4d3b8257842be18b59b4", "dsid_7215a3f9e9434b8390e5178ca520c950", "dsid_6914aabfedf04fc582d821388dcfbab9", "dsid_6b68cc690aed407bb180d40e8640a453", "dsid_9a7191d7761642c8b2553184b5664677" ] }, { "question_id": "qst_0288::semantic", "question_type": "semantic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c4db696d170b4c38b80baa91a45bd4ae", "dsid_e1dc5dc4231243bb8df5b9d17b2318d1", "dsid_a4215240075647cf9511eeb70fc2564b", "dsid_c2ddd5240eb04e16806d82103f7646dd", "dsid_e78e04e8afd442deb3718d2612368a0a", "dsid_8cb075307e1f45adbfe4aa4b39b644a1", "dsid_1835e03ef3c04dd1b3893ad2f951a7f1", "dsid_64a76c731a704b7390dce7f4121fd93f", "dsid_7205e01bf69240a48451b947589dd52c", "dsid_2dbb1c32a9694a97a895ca66bdf0f4ee" ] }, { "question_id": "qst_0138::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_52b7f00db3e14b6a9fe4e4821e079db3", "dsid_486e86b2e9fe4a0d9b7dc210f5845ecf", "dsid_504ceeb1cefc4420add6c8b9b3ed6fc0", "dsid_904448937b7041d2b1b8faa897eb15a8", "dsid_d6b1583b57d748e08e879c7f72fec18e", "dsid_d873f1adbe654259a48b9ba497e7e4ef", "dsid_ea1a168728a94056b4897e5a9dcf133c", "dsid_d29e3de208c24ff0bcab29b545549de4", "dsid_cb4d4827c76848e696fd896a18cefbe7", "dsid_4487523091464fc6a1b5392a18e8c156" ] }, { "question_id": "qst_0420::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b8aaecf54566449095c32eb057d0d267", "dsid_5176e8cbe14d49909e00764bbc11b6c1", "dsid_c9a936bd02d740c3a528d4373708256e", "dsid_939397ccfcb94c2bb22c31658c447b31", "dsid_5049f6dbeecd403fb766ca8305d56ad7", "dsid_2b6f25233eff498688a91b0be28c8d09", "dsid_a3303a13cb834f8bb6092e3e3febd30e", "dsid_7823c3b1f5944f2f8a540845bd7f1c8f", "dsid_25d4b562d1a34bb99df6fcbe8a7574dc", "dsid_22d01afde68240d998b86348da9e5fcb" ] }, { "question_id": "qst_0462::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_df79433aa56c489d93efd96e27dc1fc8", "dsid_e62a534321a24da2a93a003a9546804b", "dsid_b6fa323be1184c63a0536502438b6bdf", "dsid_c6073f1206c8467f82d37189f02d61b0", "dsid_cde4dd69fde3485386f8719b73bdd2be", "dsid_3d2ad970b6fb416eb019c4bc6c3c9370", "dsid_e41e27e6f3074276a0dae938018f8998", "dsid_63f48b9ba0fa42a7837c0923608a17fb", "dsid_cbe7722f2e1f44a0b96eb044213006f3", "dsid_2b9bd971016440f2b0415dea55dca1bc" ] }, { "question_id": "qst_0236::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a69d13b41aa2432fb0f59f20e22a773b", "dsid_a9bb70a65894469ca2cc97a8f9dfd7e6", "dsid_6a1da8c34c1142058ab69c3537bff385", "dsid_4ccacbb37f7c4d649d01b5d8ce04e8e2", "dsid_e7de5a80bab945c5b943d8da0de38ae7", "dsid_74e2046791184ef18f5775ddaf563383", "dsid_a5a78bead6c74f1295db150d78901807", "dsid_cab05f63321746d69df0a1e79ce31cd3", "dsid_2f4b42ce326a4b39a3558f97a9f8fedb", "dsid_08dfca1e608a413fa3b4c9bcbefe3f3a" ] }, { "question_id": "qst_0464::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_199789e68ef741bca44a24d535ed6c00", "dsid_9629e3c800de4029acbb92d2922c2e52", "dsid_afd51145fb074909a47894ca1f581ea0", "dsid_c1284a3217fe4b2f8c874471705cc47a", "dsid_21a09f849ab84c38b5bfe145f0ff2fa5", "dsid_416ddd531ae74684846c64bfcdd08798", "dsid_c1f6b8ef13394cc8a1e8c6fb5d470c13", "dsid_18a7896fa0dd446984d5033199aaa19e", "dsid_363a54fc4654416cbfa306d3f3f51dfb", "dsid_8371546454ec4c71920e413d2cc3f059" ] }, { "question_id": "qst_0373::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_16996cd96c1c4fa1ab399cbc446b186f", "dsid_9b30f5e9d84346fda6017553e8a78932", "dsid_5324918997644ea090703e088b166214", "dsid_7827e9b44d394a20932f0e4fb0c677b0", "dsid_9519a6d83f8c4ba49ab30e535570efe7", "dsid_5c8526ab396244cda70fd5a666fd5a11", "dsid_0771b119d60346c2903084d9069b6fe3", "dsid_5217b3a64c4a433c89ba2a3186dea82a", "dsid_a8bb0a59e5bd4279969a8378675672ac", "dsid_989b00c7aee2497285a43c4045e5217a" ] }, { "question_id": "qst_0382::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7d347905549f4456b0d2226b47e4aa68", "dsid_6db1f3d782934a539fcfe790002b8383", "dsid_6f57654840b840ffb453ab59198b4bfc", "dsid_94f03d3be70e4590898fb4f6e20b5d44", "dsid_408a30c101db449a99c79d80e89431fc", "dsid_d3584534291940eda0232157949b0f16", "dsid_d04dea2d736445c3af6934b900a12ef9", "dsid_1d3d721d88ee4f5281e559b0dc1babc1", "dsid_c1a88137436042189c6fb011ece2b760", "dsid_c8243f898ef244adb29c5e064d7e3ec9" ] }, { "question_id": "qst_0040::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a1b055020a0147c4acbc4e18682b154c", "dsid_43ecb807903f46848e8d099b6c8beb04", "dsid_423906d961ea4729bfc4a5e100476587", "dsid_6b5e244611784255a6a7328dd45a4516", "dsid_b73e360498a74c1b805feb86aca63098", "dsid_e51021765c694a65bd60b3d38b12db6e", "dsid_5a060c16f9e943dfb6556b74d1bee39e", "dsid_6c312ffb9dfa423c97d27750100e63d4", "dsid_c867e7640b0144a09a6a3823d6b6457e", "dsid_8c83fb764fa54816ba3edf3e43d496b8" ] }, { "question_id": "qst_0323::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a2c7dc70aacc470894c953b6c7499228", "dsid_22d4b181f7ff4a06a983263de150bfb2", "dsid_5a676c302f9443b9b1238f6b627a73d6", "dsid_975008fd748a4e6484129ccad7cefde4", "dsid_c91ef443a2c44aeaaa602bd031de835c", "dsid_dc2cbd6252d54a078187c00da64216d0", "dsid_1ed930c288a44ee7baac03df0df26869", "dsid_78a2276db4be452f92012c84c79ada9a", "dsid_41f825d6398e4a6c930f2d1f6de7b86b", "dsid_0feb64a2a0b242458a41ed291b393cdc" ] }, { "question_id": "qst_0003::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_c5288dd4874345adb80f4a71c9a18773", "dsid_03b2db544a6d4cf886fb9869058a05f2", "dsid_b06feeb73e3e4175a981a645af353aaf", "dsid_8aade668aa884360bfdf5a65cff10fd6", "dsid_c0a3a63c0d2a4eb6bbb2a30db7c571d3", "dsid_fbcaabc304304f7c95d8680eaab18777", "dsid_b3b35c6e716945d39940e92541c8bac5", "dsid_6e41c4375a8b4338a952ab8988ac3e5c", "dsid_44f2853bb4594e01b6fcfbba892d4ed2", "dsid_9db5a9acfd274043beb71a6308df8546" ] }, { "question_id": "qst_0150::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_8cae78271e1c4cb5a14eed6f4b074c85", "dsid_029e9c3544a54c0884054d84bcbcd9c9", "dsid_56ca7f92f5194deea0f8eca06aa48f72", "dsid_e3372197f5dc49aeba00093e195ccc8d", "dsid_8f8277de59954050b838ce8956c33a54", "dsid_a75a4d9aa7d940009b7966188753d520", "dsid_256d0212e449491696226972dc40a954", "dsid_f5ddd650161c4f2bb6d16cfa4691bd62", "dsid_a14c8d92b527408db3fa75d7c09f5ea6", "dsid_5cbffdc3333b4a198f07726cd42f683c" ] }, { "question_id": "qst_0134::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_97b29b97af2143e7893deac7ee174734", "dsid_cdf9b20094574e8eb84dfc5f8a1169d6", "dsid_c42820ebbc9f4ad8ad04f2c9d1fa4faf", "dsid_6b02cc5ed8534e35883861be573070a3", "dsid_207f2f98c14e469c8bbc88be8f328f0a", "dsid_04f86f0e609d4d14b978b23d3fd3f271", "dsid_c8c8b3ec965b43c7a88f791aa70778d5", "dsid_222b0aa878e24647a829d8cc0d144e73", "dsid_403e4b9c83a44e4bb5addcdec5a747ef", "dsid_d919fa3f00914d76994bb05e0c7830b8" ] }, { "question_id": "qst_0453::miscellaneous", "question_type": "miscellaneous", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a8a5eac6e7a243319426769963506277", "dsid_623af16bd77c4e23931de9d9e70c7daa", "dsid_a254e962c4b64a159b47faa3c8758592", "dsid_5e3b7bb4753f4419ae8814599c9c5c53", "dsid_7b73407ca91441af9e7f69c425065895", "dsid_25bf55b33a9e4d29915b4c61dcf5eb49", "dsid_e74521446b8745d596d7787dc31ed09b", "dsid_43e140e327b847c5b8571d53d8cbff59", "dsid_bd418f9ae9ff4c7caa4f33f79379d4f5", "dsid_8c295ebfc8834f9fad53046a6d78593a" ] }, { "question_id": "qst_0423::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_b8a286c123f44464b86cb4b1faa8dbda", "dsid_d417b614467040e0b9f40ab37dece165", "dsid_dea9de3e5e904e4aa4e173e254e785c7", "dsid_4a2476f041294cd6a9fc0344db20a553", "dsid_dc1575ba551c4cfd900390a5db0a6cb0", "dsid_8baa852c57644c989b430c2f749b0cab", "dsid_caaa763133944643b120dd51469feab2", "dsid_c5f0f39c0d2e430680b1b12b523493a4", "dsid_d7e5684963c44d0cb19b2a4ecc84b530", "dsid_b0e1344ddb2741d8a985032aab472179" ] }, { "question_id": "qst_0038::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_d04ffbe63ea44224892533eb0fe95bc6", "dsid_06e5984cb6004090a898923bb50635ba", "dsid_ed463c3acff94c5d94e2959eb955d84d", "dsid_685dd9516ce84945ab84837cba7b980e", "dsid_8370a3e01ed54472bff4e036ab88a53d", "dsid_fac381191235452f94e8f5882942ff47", "dsid_3ecc12bb1262430697c998b43bda1bef", "dsid_4f53cec258db4f4bb6a9594c752c9c38", "dsid_5ccebd61ccd84ca7b392013d3277002f", "dsid_d29fc29f3b7941e880b5004f32a4d07f" ] }, { "question_id": "qst_0392::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7fd136dd100c4c9cbbe1958c94c26152", "dsid_03fef9fd76bc47989e476fbe7f11a36b", "dsid_5bfed526d8144b0d8adfdd0bb5dd5419", "dsid_ebdc776e4fcb481fbae68e9d7d5c767b", "dsid_5aa1223a107c4d7bb3c331a213811678", "dsid_dc5f484a1a844bc2aa450f949a449ed8", "dsid_258679d566634d4092bd5393f349ad04", "dsid_f8e543505c1b441aa5ff55ef4c7a1618", "dsid_65bc45fa1991455a9dde2c1aeff083f6", "dsid_265b811cd2834a31a5be5bd20d425754" ] }, { "question_id": "qst_0091::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2f5b4312d3024ce6b3661baabead25c7", "dsid_59f0e609869142e79a0fc487c14f3819", "dsid_65be1880df20486fb86448f27287ed90", "dsid_e9fefe80f3064741ad632adb186aebb7", "dsid_7aa18aaa818d4d8b98491fe541723548", "dsid_6f2258feaf6d4215a50867a9b0e2961a", "dsid_6b3bf537e0b54d76948f36c106333492", "dsid_fcd51a633fc943a5b7c4d870e08caf81", "dsid_9ccc66cdbf054262b93d32bf86fd9daa", "dsid_29f8e5a2cd3247a48d75724c3326565a" ] }, { "question_id": "qst_0389::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1d53445cc74b4d7db325cd585736f87f", "dsid_4866ef84859a43b89d77178c28d9513c", "dsid_f0a7b58159aa42b791b955ddcc4b8c1b", "dsid_48ba75d82cbb495aa94c71edeb3384df", "dsid_4688847e3fc04d57afbbc92d03e6f497", "dsid_456e37c950de48b49ec504feff97964e", "dsid_a9d291a78fc749758e972678d5f667e6", "dsid_a5a5fbc9d01a4ca396d77b2a5212e572", "dsid_7cf7259bd99a4ac588aacd521fd1600c", "dsid_784e529eaa494f65a3e73441506250d7" ] }, { "question_id": "qst_0045::metadata", "question_type": "metadata", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f14dd150be81453e9b174a68967e3afc", "dsid_1de62e49b76e4f1e97f539c3b828fca5", "dsid_9ee1a08a0c9b46e3a15efbe7d6b3b3a9", "dsid_80a4859760ce4cf4af51f7c570903ab8", "dsid_697554a81df144b5a25451c72257f5e7", "dsid_86e6f9faf9924feaac3f5a88b1a04dbd", "dsid_811edea594c941c1ae19d0c0d43dce2e", "dsid_8aa30532ab3f4ed5adfb7e9ef0c9e808", "dsid_c7cdeae220704e28babfb27d8c19f8bb", "dsid_a94ae5ae140f4127b1b19a94b46ac072" ] }, { "question_id": "qst_0415::conflicting_info", "question_type": "conflicting_info", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_f106118d8ef045c7ba836a0aa8fc15b4", "dsid_a60c963f454e426da3b193511e521742", "dsid_05841423a55e4c35abd276d9899b7134", "dsid_7cb7a20999fa4fe78788c539c19c8952", "dsid_06f25d73e5d149d480404038db91cf75", "dsid_aaca0e219c154edda564f859e638cbc2", "dsid_2f9c99fc8ec74495b4b08c49a6083ba8", "dsid_5dcdefc0f60c4113940f84aa783219cc", "dsid_9325c2398e0943798f7bca20772e4c9a", "dsid_69f4055aba574f3680b990729fdb2a0f" ] }, { "question_id": "qst_0045::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a673ed031ece43c5bc4a0d27a284c8dc", "dsid_bd43557752194726821d5a36d8a06da8", "dsid_d7e5684963c44d0cb19b2a4ecc84b530", "dsid_a5af798449154b57aa7fd07b2193b573", "dsid_8fa2d2643c434a42abb38489242afbb7", "dsid_e5f976c021f64ad6bd4f99bf870458a4", "dsid_64a1a88a285c49d2a539988b3e42defa", "dsid_4fe81e1d7f774e649aa69869556b8840", "dsid_105a2db22401476a9130fbd74ac95472", "dsid_537cdcbfb5d747b4a88f513de1f7e4f4" ] }, { "question_id": "qst_0208::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_0378cb48015d438fac7668e261750bd7", "dsid_59dce3ecb4d34d7ca5d24a8073a0e53f", "dsid_b545b43f999a4c31be43572bfc5bfcf4", "dsid_e4193c0c5b4543b285367cf8b7e7494b", "dsid_aee15819ea04456c8ecbe0415251da50", "dsid_bee960def2d744dca5c15c6a6c739fa9", "dsid_6f8e7a7292864a9f8dfad31d06b7db44", "dsid_ef97562b283e49d28ef9e8597ddf51ff", "dsid_0cea8c4e421c4cc8819b469b853dbbd4", "dsid_8e015e8f7aee45719dd2f32edb923747" ] }, { "question_id": "qst_0278::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_4516e993ebdc45edae8e91a96e39c5df", "dsid_43dbaa6340ba4f6b8f5bea0fe2354bc7", "dsid_8791af9c0dab4eb9a0d23ad929980b07", "dsid_f6f389b9e1d14fb7a84b94004494b3a0", "dsid_41177e0cc51840dc84ca0ad13ac42486", "dsid_fc5a193a6d8f4e799bf912e22c448082", "dsid_63f6f5edf1cb4ed4bd6ece9fe955bfbe", "dsid_facf9ec08cbf44f3b69abb00a61510b4", "dsid_55d1a85484e043bab976c66f63aa44a4", "dsid_028241586f6f4f28842a033de100f752" ] }, { "question_id": "qst_0189::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_62902788debc4ccf96fe587a3d0d0fbc", "dsid_bbd44554aeb34c2b811c78c1b7e382c1", "dsid_8197be9fdcd74849922b03acf7b1ea7e", "dsid_20ed9182867145709feac129172377fa", "dsid_d5f59153d8fa4df8a43f66c4b1bfae6a", "dsid_3146220608b449e4b3a6f543f9761dbd", "dsid_29d203d36cfa41ba9c2ef16023290441", "dsid_0ddfeadf48294af1835098a103a4c9fb", "dsid_e515e16fbeac4e129b662d5458b1e8dd", "dsid_a479c3f232164560932ce51013333379" ] }, { "question_id": "qst_0078::metadata", "question_type": "metadata", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_cfc93aaf69bf456cb669f5b99c09b7fa", "dsid_b9e31a2358c7454ab25d7b424ee8284f", "dsid_fff8b5690dda4d7989b20588972c5f86", "dsid_cc4c5c2310714fd99962514beac82e74", "dsid_5067d272242744b2bdcc63c8467c465c", "dsid_8e29f4595bac44588c825be6b9113ed2", "dsid_f9315673354f409b85626ebf4694005d", "dsid_d627cb5b7fd04236b3fa0e0aff2f39d8", "dsid_8f6696d472574c0994c64086775c9dbe", "dsid_27837dbb082541f4bbf83a06506f1f1c" ] }, { "question_id": "qst_0318::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_521553a072244d199d06a34dd6e1aca0", "dsid_d8cbe3c373104b91879d2da05b7bcd6d", "dsid_0aa853d7800b4ed7aa2779e9eae97c63", "dsid_681485e4afe04afe994f922b21517983", "dsid_d363f62f0c84438b8ed697891caf6471", "dsid_51fd2c83dfae4af39a913d2cd07bc369", "dsid_d1c65a017466400e9a891bd531b87874", "dsid_3a9ce3f606694773b3044a556819cb31", "dsid_1b20408d6e614d6aa8e134f5bb23ce98", "dsid_e92b657f618445a082d758b61408847d" ] }, { "question_id": "qst_0003::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_3fd6af404fae48e6b8ea5a57875ef78f", "dsid_f89cfa7dd9214a499571192e1d28b2c8", "dsid_f7cfbe7b9f1a454cab1815b3dd2430c4", "dsid_841f64eef86e410fab68aa14f64ab949", "dsid_ca110faf428a47408721cdc464207295", "dsid_79a7e4a91af840569edb92f50c75fb06", "dsid_300e26baece74bcea5eaae36496fa8b9", "dsid_6296a03da6864e3ea90bf0be892bfe69", "dsid_ef5eefa2ddaf4103bb7e810b10e9c95f", "dsid_94e59147824b4d34bfa1cb418e2731c6" ] }, { "question_id": "qst_0197::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_c9cc1fb8593846a4b42ddad7f5cb8c99", "dsid_f1f8c246c42d4b9e8030b6320d6fb0ad", "dsid_6e2ea493db354c56bc26050e00b2dd2a", "dsid_a559687e57f24826b4beecfcd6322d48", "dsid_c03c9a3a983d4fe88ea4e2054c9e9fa3", "dsid_ceb5472c6ffb4fa89a7a0a76f6d2c7cf", "dsid_d348926d7c7842149088744e3b3763f0", "dsid_331099dcdb6346efa20bdec9f477fbd5", "dsid_2401e6c7889b4c2081c66dae551f434a", "dsid_158f9bbb596d4f9aa267d5ff4f24513b" ] }, { "question_id": "qst_0041::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_7351bc1b2f924a6a9eb795cb9f1d96e8", "dsid_dd52544fa3da4baa964e521152da1a19", "dsid_0ba04ec017364392b4d5221b1c0eb134", "dsid_56d94079de21420592ce7e1553800a27", "dsid_8ac31912d9974012a887f1ffa5c48ac9", "dsid_5d8cdd5aa3d74802ba490573ecb13937", "dsid_46f929172d9648d09066b0f9b4069bbf", "dsid_3e8cf249680a46a58bddd1faa893c1f4", "dsid_b9feabcc616a441dadb2e6237166b316", "dsid_1c753daaecef4c2b98ba2be0cc31af32" ] }, { "question_id": "qst_0166::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a110cebd6e8844afbb800d73e451bf55", "dsid_99b157396e5e422090dce29d0aac8ea7", "dsid_d47b69665d364df58ca39f83784544bd", "dsid_edb28b72310749718834d4e9f7083bcc", "dsid_4cdcb62bfd154b2294269faedd815607", "dsid_298e706b9e2549b9a33776ebfaadecd6", "dsid_29a2eb3d8d974d18b4560fff32f361f8", "dsid_f4a17a364e484f02bca535db4c570b9e", "dsid_5be846b5dc5247348dc33a60aa9b0b30", "dsid_3ebb269f9c3841fdbdd14c376311e299" ] }, { "question_id": "qst_0251::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_7032728f84204589803d0ee2cc72ac84", "dsid_080084005f9c447ba4a3ecef1627b027", "dsid_4577aa8385ab41a29cb33b8032c07c83", "dsid_447b145145ac475da57a16ae41af5f0b", "dsid_1afe209ed9084bcf89e7f2d44cb30d10", "dsid_45c7d44f740e4f6e89e589e97cb0b0d6", "dsid_c45f5dd389cd40228da811f7084edbe2", "dsid_5c77351359f64500a0fcf0d558dc1064", "dsid_7c73ae54dc804f99b5779bb40d20b6b9", "dsid_33dfbd7e6dce429ab71aa69998f63284" ] }, { "question_id": "qst_0085::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_2271ae6c393d409093b11b5a6d5b96ed", "dsid_c94d48a530b4477f99630e9e10b54703", "dsid_0ac29bc8d5cc4e11b2b41a193f3731e6", "dsid_523df3a44bee447887707c0aa1f72996", "dsid_dbacebbfca96474a91249e74e54605be", "dsid_c30d82f87440478c9a75d672471fc1a0", "dsid_f7785a67900d432584de232f2a309040", "dsid_d9f77f614f174eeab9e2e48dcc859b5a", "dsid_602465147b7547d8b72599e2d65f09bb", "dsid_e84dca82e517439c8773d80f7977758a" ] }, { "question_id": "qst_0448::completeness", "question_type": "completeness", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d8ec0649c34449d3bc48f7a43cf62372", "dsid_e51021765c694a65bd60b3d38b12db6e", "dsid_107b32582f064c0481b84fcc630e026c", "dsid_6c57c9d4e08947d6bdd35a0eb9bb7c6d", "dsid_b6fb2620d47d44e3b17b16b42f1e021f", "dsid_f7d5fea86ebc4266abc8a92f79b93565", "dsid_d1a5894a308e4737a4c1a591e962d337", "dsid_5a060c16f9e943dfb6556b74d1bee39e", "dsid_f5ef1167ba894d4db274ce35219e1f3c", "dsid_97200e6e3c2b4062a8cb3c13047a464c" ] }, { "question_id": "qst_0029::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_429267fe79c74918a26272f6c0e48f10", "dsid_24ad37cf41bd4d1190e6741cd5a87569", "dsid_18d6b7f9d2614de284aa176dabf09bf7", "dsid_ad4430eaf01243c2acae87cc61fc8553", "dsid_5a8502371e7043ffa2a49c103429e981", "dsid_f7ea200b105a46cab07f4e1644a6c71d", "dsid_b884113247fc45c1946e3126688fbe7b", "dsid_1b909026c9e648988713b1726e1b21bd", "dsid_a5c1bd8de532445396c7001f12a0a82f", "dsid_b11bab4f7ca7458f8248e4f6a47dd896" ] }, { "question_id": "qst_0349::project_related", "question_type": "project_related", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 0.6666666666666666, "retrieved_doc_ids": [ "dsid_1b7350404391482987b2f33788e11644", "dsid_21d6b99dc771418287d61444f373852e", "dsid_fe7a496e795b4baca997e655f9316fc5", "dsid_03083a2341344c8fb5eb3822fda49a3a", "dsid_93105d9dc2624aec85d0a968d0f00370", "dsid_ada785411ed944e7bab1d8a5a0a42e8d", "dsid_cbdca0f419034a458855c0ab687dd2b7", "dsid_7bb2163228324aa989ff958f4a84d01a", "dsid_838e2cfbd4ac4dc0851cfbb60f3eae7b", "dsid_a12e67fc3f9149b8bf684f87d57dbe46" ] }, { "question_id": "qst_0324::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_62a56f85dced458583381347c3242901", "dsid_a92fe998fc534d57a1d54d5a6294c938", "dsid_d8c77626e9eb4b01a4342d68c3d5603d", "dsid_90a4ffa9fbe94fbc81e88e8a638bb14c", "dsid_3865bcbca3404c6f8f2687268d3ae93d", "dsid_f5d9a6af61744d8d9429f90e62412ebc", "dsid_1fb0af55bc3e4ef0ac1efab005de901e", "dsid_8840f004628b4ebe85bd7b76c4698f91", "dsid_02bcc338321344ac9f06ee21fba70bb9", "dsid_56b6c392ca654a9d98ea806b5f70ac57" ] }, { "question_id": "qst_0270::semantic", "question_type": "semantic", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_7846fa5de11042acaa1147d60d58ebe9", "dsid_d65e7de949464664a4b0b796f98199d7", "dsid_33de625fcfc045dd967dbc9bdde3a8b4", "dsid_fc1807af20be4c218212ce449270ce78", "dsid_0fc532b7ebb64496ae1732e13db3a486", "dsid_73c935afee1f44378aff0361108749c1", "dsid_9f6eba20298645bdb88fdc65513fc22a", "dsid_6bc8bb1188ca451fba7b3e342ecd89bc", "dsid_886b5ee29f464520be1bf4497044b396", "dsid_6d111c6dad6b4e67936e3479a9a65ce2" ] }, { "question_id": "qst_0266::semantic", "question_type": "semantic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_415b72b7d40a405e87b9671946e894d8", "dsid_c1f7f945d5c940f5ba28e8030ecfa720", "dsid_7bc65118ebef4a46951fd8c1ed24248c", "dsid_e88374ee23994a409f5d9947d2321fa4", "dsid_ac5e40129fa2444eb1ca1c7aa057bfe2", "dsid_88b39d13bd584ef8b1422d76994808c7", "dsid_d849fc50d6694f32b6cd04adaff35fec", "dsid_bdabd4d1a5ae43d1b53e63583847d50b", "dsid_ed4b40d5f4f746828b3644a44a9c82ed", "dsid_6673ae3aa9854e0a8c122c0d843917d2" ] }, { "question_id": "qst_0247::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_2e3c8adbe9454a258ebc9b84cb4a82fc", "dsid_2167222a219042b89154a13bd7903328", "dsid_1065770b54f64740b3c06fc89479d4d0", "dsid_3c18e45fbc064bdeb056d94d621846f2", "dsid_f2683c74e5d949958b6c04aa9bd2d706", "dsid_ab8638b600884c9dbac0c7ae5ba2560c", "dsid_e9548ab66e4346c9a328b8990492b8f6", "dsid_8c7523be45234ca09679cc17cdfe9253", "dsid_46304695ee8144c7b48e2393510ff03a", "dsid_48745287986b4d5fa562184c78d46bcc" ] }, { "question_id": "qst_0239::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_caedab050bc54b8fb63f187b5c9ec380", "dsid_11adfabf6a1046ec9ca1b761aca1ee8b", "dsid_e2367055432f494e9ef42529706f2044", "dsid_2b1e02466b1e4dbaa5bf6fe765c0cad4", "dsid_fdd27747f27949f89a9f84fe6c2aac59", "dsid_843d5646b5a543039c500c21a5e0c6f0", "dsid_1ab6c34e57574d8e9c70055e5bab1325", "dsid_3260fe01c07a4377a859ad3535073a45", "dsid_f3f5dd4e8b37472da09bd1ab0bee00b4", "dsid_a594e90ffe31433da48f962bd261f525" ] }, { "question_id": "qst_0381::constrained", "question_type": "constrained", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_aa2276f167c64dacb50aa912b2e72794", "dsid_865449ae95b044768f33773263bfeaad", "dsid_6bc8bb1188ca451fba7b3e342ecd89bc", "dsid_d844cc30147b49c69ea5275b4993b571", "dsid_ab14ca7029e344999d9e832d24939cae", "dsid_8502779782a741e4867fb5f0b5a0f142", "dsid_ccc97e282beb40fe876348533b684cda", "dsid_4cdbc8b290fa4e8abcbcb68f9a028e15", "dsid_dc578cb27bc548c7b71a7d5b2e6dfb5a", "dsid_bc85490b593e467b906db5a6336101d0" ] }, { "question_id": "qst_0393::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_90218d9a9af84072bcf3ff7bb90b5fa1", "dsid_4b15b253ca294377a12d12651ada2168", "dsid_7fc02b1c6fe446abb194bd51e7858b98", "dsid_bd728fb725cd4d2bbad33eb2eb1109a7", "dsid_e664488f72374064bf762d16e3540fd0", "dsid_65544d4851e74bfca62998f699271db0", "dsid_e8c5a9c5d3334b2292559f3b955aabdc", "dsid_dea82ae86fd64753a0257c4c8ed675a4", "dsid_fbdb90b68a2c47369bb5011328e67ec2", "dsid_ff7f4962ebbc45b093322dd70b17af4f" ] }, { "question_id": "qst_0023::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_576b502053d74913a254e16a4e8b7ad6", "dsid_a0465fe4b5ff4fc69cfbb12b2a7b8cf9", "dsid_be15e983940742569d406039294dc7f4", "dsid_c61ec17996f2484995a080938542d765", "dsid_318ff08b63c8402796be3c09a8d65d58", "dsid_960387549eb2472284385010b70a91ca", "dsid_92543b5ba7654cac9dc1bbcf0e43c7a0", "dsid_12852e4f14ee42df8d2a81d51de54d56", "dsid_f91bde5f73264d78b1cfaf05c6025d99", "dsid_f02853694af44d78b6499a3347e8e6cd" ] }, { "question_id": "qst_0380::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_be8d2802cf3245088a7e7f861f317b31", "dsid_2bea8116212844eca26c9c90052b0ec2", "dsid_f1a3e4d11e274a22b0d5e646ad0ba9f8", "dsid_3e7cf5be42d84f2880858c726b65998f", "dsid_aaaf6ae3258d4fe2a6883f2cc58e7360", "dsid_b1abbf0af864466dabf769efd7489b7d", "dsid_089917448e654911bd186650ed12feb9", "dsid_7fb28e1d5b2944bdaa45ae8894eb8729", "dsid_b08cb0b4a55d419d933450cf8d457928", "dsid_ddcb8d01dbfe477d918319ae7fa428d9" ] }, { "question_id": "qst_0124::basic", "question_type": "basic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_99a72628a053445c9d5d6340eada4385", "dsid_30a16502953a4088b59266c8990d52d6", "dsid_1ddfb064eec749e49362306d4256f6c4", "dsid_f093fb3a4d5642449c853d55f00474f2", "dsid_b7f556eb20b74d33a653ae7d7e605d79", "dsid_fc01b9470b5b42d4a55c230b13384635", "dsid_3e55ed36d4fd48a389f5291a3d1a4a11", "dsid_6d08c84f68c04f729c0b6252f0dc131b", "dsid_3e9420a2663b4fd0ba31ae7cbde2884a", "dsid_22b285b78b6a4f9aa212f118c7b1db7e" ] }, { "question_id": "qst_0144::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_034f29b60aad4b80a29d9a085e5002ae", "dsid_1a4f379bd75e484a9d28194d2b643179", "dsid_00e588c6c3284c92b86862a02cfd2ade", "dsid_0a3977d8e5004edd9bbb851daa884bab", "dsid_243ea2030db84170a5025ea6b928e742", "dsid_f792e2a148d84e9cac998b2cb5d452f6", "dsid_87e5cd75fb97443ab381d440e955d870", "dsid_8851fbc3493042198261aabc3f7c1fc3", "dsid_a7f8d8578a5e48cc91151bb692a34ffe", "dsid_5740f94238d242f4aee4ee319332ee4e" ] }, { "question_id": "qst_0444::completeness", "question_type": "completeness", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.5, "retrieved_doc_ids": [ "dsid_117e5a150aa94adc86295786bd6b7843", "dsid_5cde26f232454906879617dab1800fae", "dsid_40471ba4cd3b430eab3ac0877eef0972", "dsid_7bbf556ce2da47829cc311833ca89cf3", "dsid_b5713fed031f47318b17be5d2f62237e", "dsid_c2fc42b757d94ee985a08acdf62af71b", "dsid_2ad49b09e5454bc1a2139c67a66c56b8", "dsid_8a7fdc9812b5489f8152526545e444ce", "dsid_50339fa427954f27a8802fe110d29ff1", "dsid_a59d4d46470e419899fd289f04ceb6b2" ] }, { "question_id": "qst_0359::project_related", "question_type": "project_related", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 0.8333333333333334, "retrieved_doc_ids": [ "dsid_617d538c79334f4b9ddfd5c50c05ee4b", "dsid_d67b4ea2d8834799ae617b1ccddf1f6e", "dsid_a66ced6d922f476985e02310626aa9b1", "dsid_5b0c65ca162a43a0938bebabe9b1d53f", "dsid_eae4cd799908469e9c4cfc855897cd06", "dsid_2d0330be987e4163bfd9c42c69bc1e40", "dsid_e6a60478259749b0bdf7b955a5647e55", "dsid_4c5e26738ed141f382550d98bbcc8d23", "dsid_ae455bffe226412c9973991ef2d34a72", "dsid_f4687c1785c3441691f357e455f3010a" ] }, { "question_id": "qst_0131::basic", "question_type": "basic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_292928b219054ceabfed42b8a4689910", "dsid_2c9d01fc90cb4c3c8b0f5a73ed8bb070", "dsid_eb36744e4cb34a6d89cc33e68e1e864c", "dsid_6cd47c02e38742a3a2295baff4aae2ae", "dsid_2a4b78cc33e34dd0b084d3c148ba562e", "dsid_24f6523c83ee49b8b16bad04d091f7af", "dsid_1d3bc5d7f88642b5a78db466aa6b4b4e", "dsid_40fa8ea639f74b919eb833243cd11bb7", "dsid_45bb3438925d4726b6d0c342196d18b0", "dsid_e95b1d81126d470ea72e5ee55e454eb4" ] }, { "question_id": "qst_0449::completeness", "question_type": "completeness", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 0.3333333333333333, "retrieved_doc_ids": [ "dsid_5aec783077bc4240af93a86156a260e3", "dsid_e29a9198a5754ebe85cab10c9416e937", "dsid_5cf1a6657f1848968b2fd154066950ae", "dsid_ed6d086c9faf4272b0ee20d3fa4b6180", "dsid_9117e8e14e4542ec8e75b80210ca5c61", "dsid_ab4664f39ea64fb48c7a924fb2ed3a4b", "dsid_8dff963a50774cdd84131d9ae5e4f4f3", "dsid_4622fb99a23948beb879565d2162d94e", "dsid_762dc1934c9a4e8bbeb7e28e9f87c15f", "dsid_f1b7863b01c64a7bb49f8b8cab133200" ] }, { "question_id": "qst_0162::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_a3fdfc45905f46f3988ad13f55ecce93", "dsid_c9cb551fa9874c0db378a9936017740b", "dsid_d00f7d1c4cd74b41aae944185a56ecb9", "dsid_3042eea9b36942ae8084eab2f28c3012", "dsid_4baf344a842d4053b6b75c37b542b83a", "dsid_5053cc95922a48d08b31014fe7ff202b", "dsid_f3539272263d4896a2f5ade8725b11e8", "dsid_a18bf0f985504c4d909ea45becc7b11d", "dsid_3e413744bfcb4f2082871da734e6f96b", "dsid_0e562e8a0f5a45af8d00b0a6c4d886cd" ] }, { "question_id": "qst_0034::basic", "question_type": "basic", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1867cdf06a00437194b3250c75321a92", "dsid_5e087d4f8c8b4a238c98bbfd9bdd7a50", "dsid_85d3f7f2294c47d4b26b214fde58063a", "dsid_634dd9a56b8c4d4386c4f87b53ac16b8", "dsid_dd03b7ca67494e3d9d6c34bba9d027c3", "dsid_7cf7e11b5e724b2aa36ad8575cdfa226", "dsid_b77fd838edd94ccaa3066491bff54b4e", "dsid_ef29168dbe8d4eb8b477fa55b95cac66", "dsid_cae478d5edbb452cacf1b99d234fe4b6", "dsid_17ed1472bdc44624b874f5c3bba1b980" ] }, { "question_id": "qst_0238::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_81ea322b99b64268ac6f98e4f562c848", "dsid_ddab7755448e40128e3882c8b0545fc3", "dsid_befd7deb82634db1a30fb9821db1ad1f", "dsid_92fcbb73d7b74592a742c678fdd521dd", "dsid_e06bd9413e5f4f34b9b04eab2953162c", "dsid_be288f0682c24ffe8a6d3fcbe7696932", "dsid_2a8f6583188d494b8907aab8e151cf38", "dsid_084adcd20d7c4c76855b9a48317e77ac", "dsid_4f22a4d739c14b3ca4bc6a6bd5dee154", "dsid_55e3f1cb083f48128c3aeb7539b3e287" ] }, { "question_id": "qst_0057::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_026bbc0fbb97423ab9de20d8f99402d6", "dsid_d740645aa8ce42059a6967bdad866b82", "dsid_718a5c08e1ef47d38634c3623ace0803", "dsid_3c368798ac95482a8bef9e28b6f4af96", "dsid_4688da04d8b549e68271991245290a3b", "dsid_e9c5246734c849a899e29c60e6134988", "dsid_ea08c39b16de4b76bf74ef89523333f2", "dsid_3247b6d0a4e84fa6accf2789b28efa23", "dsid_5cf28151bea84f5b9736145cc853cf83", "dsid_54628cf1597541c2878708e1c2e511c4" ] }, { "question_id": "qst_0268::semantic", "question_type": "semantic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_46b0bc1e97ae4d8792d08e6f4122b948", "dsid_d4319ac5979642dfb0612c4f35cd4668", "dsid_42d51fc5cb7746e4b1c9200136795a36", "dsid_a990e708840f481795741cfd3fb55691", "dsid_9c62c96a8d3c4a4fa67b4b61d58b1806", "dsid_32d162435ad34eeca3861eeca4463879", "dsid_456748c914da4e7b858c7bb1dfa719d3", "dsid_d885f7fe13ec469d8b6ac95d64c2fcec", "dsid_5b35d075a1c24243aa9e542eec17c5f2", "dsid_618649e367d24a1eb3bdcacc0a2d7da7" ] }, { "question_id": "qst_0261::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_fa79643c4fe4430395363b868d306267", "dsid_6202c704d54f4d8e818dccdfdeb3fd12", "dsid_c9a1ca32dae14498a89c2ae1b1d5bfa8", "dsid_1245cd2b544643bfac3591a78760c7d7", "dsid_4c101d6f4fe64b5785607813597c989f", "dsid_750c92ca239644b4a8f3bc0a4ac68087", "dsid_e8db388ec62e4357b0b91e9fb6b82ff1", "dsid_5f137275c8ed438cb7edc9f1d8740ddf", "dsid_c6f16a3fc2e84f2597dc0786e4f16ae0", "dsid_5fcac9f223fa49b58cb327d7c32f99be" ] }, { "question_id": "qst_0021::metadata", "question_type": "metadata", "rank": 3, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.3333333333333333, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_1de730e77fc24861a1c98ba2afef8be8", "dsid_0a3977d8e5004edd9bbb851daa884bab", "dsid_929598314f834ad08cf48642b88bde16", "dsid_ddbe235d0a69489fae2809702b2e6b24", "dsid_b62f660578c64e7daae4baa0c6b64e8b", "dsid_a9b2f36e07214d19bdfee59c64d36e2d", "dsid_08335fcaefac48ca9da2e5b764059690", "dsid_e31c9f4814284767820cabdd7ea7774d", "dsid_d05cb3bab0b049c1a0089f3b187ac890", "dsid_05cd86a55fb44ee1a48a0f8066b36e8b" ] }, { "question_id": "qst_0298::semantic", "question_type": "semantic", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_d4b2d3e204f843c0864d77809f1b9881", "dsid_94fff4908fb14a8eac5eafc25fbe7f6b", "dsid_9e132f73268d47849813d8218a4d06f0", "dsid_e8e41ea4c8af47f4818fdda98ec43175", "dsid_9d471c0660bf401da9fcbd222bd889aa", "dsid_91321ce975ee46ee971ad9988db7d776", "dsid_408c94bb372743f4a622da1d30717c83", "dsid_c11687783b394fa2abe8d5b4a4db27a3", "dsid_3ace0c115f2a4068b73311534e0ba8e1", "dsid_e787c48ea74e43968a6dbb4674d4db86" ] }, { "question_id": "qst_0313::intra_document_reasoning", "question_type": "intra_document_reasoning", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_de02efca4c804ed8a31dacc77a73b769", "dsid_e9f22ba025e34bb6b7b8c61d26a38a19", "dsid_30ce3339ae2c41e8b5011ae8e824ef8c", "dsid_7f2a6f2d2de34f94a5b6f3d5d0d1f6a2", "dsid_0f789284a1514713b7fd67f4d810d258", "dsid_b4f5ca2611ec473c8affea7e0fcfe4bd", "dsid_87fb9790b17347268d4e0672af499c40", "dsid_72f9a72f30184cb8b1093389c0c6a8d4", "dsid_fc6bafced8dc48099ca403957f7009cf", "dsid_1988feff8a414541a33ba3ac70400ac4" ] }, { "question_id": "qst_0244::semantic", "question_type": "semantic", "rank": 2, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.5, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_ca15f318846949608e88005cc49a5a8f", "dsid_37c24c697d9d462fad0e3afeeea9d8c0", "dsid_3c784dddf8254494b0355227acfd14e8", "dsid_85cd5df03df744d5811b1f0a388c4652", "dsid_8e85c02011ff4ddc92ff86bcfa3cb72d", "dsid_3df48dcb80af4806a88d4ed0f9b04c30", "dsid_42e93906357541fd97d56f5f20923a9f", "dsid_c3ab94fe217d4dfe9c6e0b88a40cb6dd", "dsid_372fa164f5814ebe9adffb584600d9b4", "dsid_af4858ff547a4246a84e385db1c21aa8" ] }, { "question_id": "qst_0077::basic", "question_type": "basic", "rank": 4, "hit_at_1": false, "hit_at_5": true, "hit_at_10": true, "mrr": 0.25, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_6b5572d3c235497bb5e49ceee621840b", "dsid_1aa1420618c0424eb1ccdd38fce758d4", "dsid_1dffa4bc09594854ba264a78145d4dd1", "dsid_88b39d13bd584ef8b1422d76994808c7", "dsid_9bba995193d24e51b9484e66d919fdff", "dsid_b777b0c6d20a47a698ab7c3c2b92be94", "dsid_5b7a0d3bd5fb45c2a2f936c886f60667", "dsid_f4e5c8b7937f4e3d8d4263ab11c1d36f", "dsid_22e5b57845114325acea1eedce0321fc", "dsid_169dca195c96410b80ff166803a50512" ] }, { "question_id": "qst_0424::conflicting_info", "question_type": "conflicting_info", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_79ce9633920044a9922b29b7f26077e3", "dsid_d9f77f614f174eeab9e2e48dcc859b5a", "dsid_d43cbff0391944e8b00d75141767ff98", "dsid_1f5f10cbdf6b464c8c1cac1f4bb48e27", "dsid_403e41101d1d43fabcd3f392ad1afae8", "dsid_f34f0f2a273d4c18b06afc7a77fdf58d", "dsid_7f6f5f0ab0e2481f8c3b03d69df3f3af", "dsid_4101bfe0e39e47208ebbacfdc741f643", "dsid_904e9c6d4fe3443fb66e7f7eccb4d782", "dsid_5e2b017deb9d4b8e8fa422b37964bfc5" ] }, { "question_id": "qst_0401::constrained", "question_type": "constrained", "rank": 1, "hit_at_1": true, "hit_at_5": true, "hit_at_10": true, "mrr": 1.0, "document_recall": 1.0, "retrieved_doc_ids": [ "dsid_f7dc91cc70e14c94a6215a13cd829cbf", "dsid_d36eb4801f674ebd8e9a44fb730b7555", "dsid_50949c6b09cb4a57a3b5a23eb081efed", "dsid_12405cd718b74077aa59affbc47804f9", "dsid_80df3a989d0e417baec286cd4c0d3e61", "dsid_f5545e84d0af43e38eebc77467015394", "dsid_f0c687510ce742b28d3655cf26f73cc7", "dsid_6b3782b8245e40f8aa6f3f69087ac2fc", "dsid_3eae2c0af3cb4533a3a540fdcd762ba5", "dsid_a82f21999d104dc6907cae46960ac4ec" ] }, { "question_id": "qst_0053::metadata", "question_type": "metadata", "rank": null, "hit_at_1": false, "hit_at_5": false, "hit_at_10": false, "mrr": 0.0, "document_recall": 0.0, "retrieved_doc_ids": [ "dsid_a3025332650949328807df35edde72a9", "dsid_830d34024db447aa8ac894b0ed042f1f", "dsid_188ba72f0e734fe99c788e58db82c640", "dsid_9440685347db4846aba0796ca75bc5b1", "dsid_09db191ac38843c5a6ee7d4b33dfcee8", "dsid_8091897e4f53468b9c1cd471f765e949", "dsid_c75060415f5a4a1594074711745d144a", "dsid_ebf2f495542d49309b6862d7d40e11cc", "dsid_36ffc3c8a15a4e5fa4ccebfdf00848d9", "dsid_e0f147df944a4856a0659aff26f05c09" ] } ] } ] }