qa_latest.json

{
  "_meta": {
    "synaptic_version": "0.17.1",
    "saved_at_utc": "2026-04-19T13:09:03+00:00",
    "corpus_md5": {
      "krra_chunks_jsonl": "missing",
      "krra_documents_jsonl": "missing",
      "assort_products_csv": "missing"
    },
    "query_md5": {
      "_public_sources": "ec3efacbfa6d9cb79a85f170f7cc02c5",
      "assort": "72f75f06e4cf69f2d4081e7f324f3c75",
      "assort_conversational": "b7fae1f4c28d47de9367e5a8676de7e7",
      "assort_hard": "905edf0690626daefaec1f062bfd6a7e",
      "krra": "cf66228bcd778bc28ea7f2a2cb32ecbc",
      "krra_conversational": "a0797c687ac4376bad3a5b19798538e2",
      "krra_graph": "6d17554a89c20b9c6af892a0124b6810",
      "krra_hard": "9310e2325fff3cb4289b35850073e710",
      "krra_multihop": "678cbbe4840c2ef5fcb57abdea133240",
      "x2bee": "784fb421cdd65f88ec396d3df02bcb8a",
      "x2bee_conversational": "c77e2498d4e5efe535d9bde77077ecf0",
      "x2bee_hard": "65adc5aa4ab9afa62895dbcd031657e0"
    },
    "note": "Baseline written by eval/run_all.py. The `_meta` block captures corpus + query file hashes so phantom regressions from drifting source data are easy to diagnose. If you see a score change after a code-only PR, compare the hashes first."
  },
  "KRRA Easy": {
    "mrr": 0.9667,
    "p_at_k": 0.5914,
    "r_at_k": 0.8572,
    "ndcg": 0.8664,
    "corpus_size": 20
  },
  "KRRA Hard": {
    "mrr": 0.5828,
    "p_at_k": 0.3243,
    "r_at_k": 0.1961,
    "ndcg": 0.3124,
    "corpus_size": 40
  },
  "assort Easy": {
    "mrr": 0.8556,
    "p_at_k": 0.1,
    "r_at_k": 0.9333,
    "ndcg": 0.8754,
    "corpus_size": 15
  },
  "assort Hard": {
    "mrr": 0.0,
    "p_at_k": 0.0,
    "r_at_k": 0.0,
    "ndcg": 0.0,
    "corpus_size": 40
  },
  "X2BEE Easy": {
    "mrr": 1.0,
    "p_at_k": 0.14,
    "r_at_k": 1.0,
    "ndcg": 1.0,
    "corpus_size": 20
  },
  "X2BEE Hard": {
    "mrr": 0.3684,
    "p_at_k": 0.1,
    "r_at_k": 0.3286,
    "ndcg": 0.3684,
    "corpus_size": 20
  },
  "KRRA Conv": {
    "mrr": 0.1464,
    "p_at_k": 0.058,
    "r_at_k": 0.0733,
    "ndcg": 0.0732,
    "corpus_size": 30
  },
  "assort Conv": {
    "mrr": 0.4722,
    "p_at_k": 0.0917,
    "r_at_k": 0.5208,
    "ndcg": 0.4586,
    "corpus_size": 30
  },
  "X2BEE Conv": {
    "mrr": 0.1636,
    "p_at_k": 0.0222,
    "r_at_k": 0.1852,
    "ndcg": 0.166,
    "corpus_size": 30
  },
  "HotPotQA-24": {
    "mrr": 0.9094,
    "p_at_k": 0.1917,
    "r_at_k": 0.9583,
    "ndcg": 0.8472,
    "corpus_size": 226
  },
  "HotPotQA-200": {
    "mrr": 0.8535,
    "p_at_k": 0.17,
    "r_at_k": 0.85,
    "ndcg": 0.7445,
    "corpus_size": 1990
  },
  "Allganize RAG-ko": {
    "mrr": 0.9585,
    "p_at_k": 0.1058,
    "r_at_k": 0.995,
    "ndcg": 0.9673,
    "corpus_size": 200
  },
  "Allganize RAG-Eval": {
    "mrr": 0.9273,
    "p_at_k": 0.0992,
    "r_at_k": 0.9733,
    "ndcg": 0.9382,
    "corpus_size": 300
  },
  "PublicHealthQA": {
    "mrr": 0.6107,
    "p_at_k": 0.0857,
    "r_at_k": 0.8571,
    "ndcg": 0.6703,
    "corpus_size": 77
  },
  "AutoRAG": {
    "mrr": 0.8954,
    "p_at_k": 0.0991,
    "r_at_k": 0.9912,
    "ndcg": 0.919,
    "corpus_size": 720
  },
  "KLUE-MRC": {
    "mrr": 0.7723,
    "p_at_k": 0.0859,
    "r_at_k": 0.8546,
    "ndcg": 0.7924,
    "corpus_size": 5841
  },
  "Ko-StrategyQA": {
    "mrr": 0.6451,
    "p_at_k": 0.1353,
    "r_at_k": 0.7242,
    "ndcg": 0.6143,
    "corpus_size": 9251
  },
  "2Wiki-dev": {
    "mrr": 0.8373,
    "p_at_k": 0.1443,
    "r_at_k": 0.6078,
    "ndcg": 0.6197,
    "corpus_size": 56687
  },
  "MuSiQue-dev": {
    "mrr": 0.6981,
    "p_at_k": 0.1221,
    "r_at_k": 0.4859,
    "ndcg": 0.4726,
    "corpus_size": 21100
  },
  "TREC-COVID": {
    "mrr": 0.7243,
    "p_at_k": 0.544,
    "r_at_k": 0.0133,
    "ndcg": 0.5533,
    "corpus_size": 171332
  },
  "FiQA": {
    "mrr": 0.2937,
    "p_at_k": 0.0647,
    "r_at_k": 0.2936,
    "ndcg": 0.2308,
    "corpus_size": 57638
  },
  "SciFact": {
    "mrr": 0.6226,
    "p_at_k": 0.0883,
    "r_at_k": 0.7986,
    "ndcg": 0.6603,
    "corpus_size": 5183
  }
}