{
 "_published": {
  "source": "docs/measurements/reranker-es-legal-idle-gpu-2026-11-16.json",
  "published": "2026-11-16",
  "publisher": "J4SGON S.L., vorluxai.com",
  "index": "https://vorluxai.com/data/",
  "cleaned": "Copied from our internal repository by scripts/publish_measurements.py. Every number is unchanged. Redacted: nothing. Hostnames (spark-43d5, spark-7180) and the PRIME / spark2 labels are kept: the posts use them.",
  "redactions": {}
 },
 "schema": "vorlux-bench-run/1",
 "slug": "reranker-es-legal",
 "date": "2026-11-16",
 "machine": {
  "alias": "prime",
  "hostname": "spark-43d5",
  "kernel": "6.17.0-1032-nvidia",
  "arch": "aarch64",
  "gpu": "NVIDIA GB10, 580.173.02",
  "mem_total_gb": 121.7
 },
 "command": "python3 scripts/bench/reranker_es_legal.py --reps 5 --out <tmp>  (run by hand on prime, GPU idle)",
 "note": "Rerun of reranker-es-legal-2026-11-15.json on the same machine with the GPU idle (0% utilisation in every sample before the run; the 15 Nov run shared it with a training job at 94 to 95%). First-place counts, top-10 hits, MRR and the sign tests are identical to the 15 Nov file. Timings differ, as do indexing speed and one count of questions with an unstable rerank order (1 then, 0 now); this run did not pass --keep-alive.",
 "versions": {
  "python": "Python 3.12.3",
  "ollama": "ollama version is 0.30.10",
  "gpu_driver": "NVIDIA GB10, 580.173.02",
  "bench_script": "scripts/bench/reranker_es_legal.py",
  "bench_script_sha256": "de73b4efbb454d86d8073c3842bf821b5a4c34c51313b3bb2543c72bc6756ec7",
  "repo_commit": "ddb7bd26d17fc72c83db4331687abb659b9959de",
  "bench_dir_dirty": false
 },
 "results": {
  "date": "2026-11-16",
  "started": "2026-11-16T05:55:46",
  "protocol": "Does a cross-encoder reranker help Spanish retrieval, and what does it cost in latency?\n\nBehind one pair (site/src/content/blog/):\n  * reranker-spanish-rag-measured-en / reranker-rag-espanol-medido-es\n\nCorpus: evals/reranker-es-legal/corpus-aiact-es-2024-1689.json, the Spanish Official Journal text of\nRegulation (EU) 2024/1689 (113 articles and 13 annexes, no recitals), cut into numbered units by\nevals/reranker-es-legal/build_corpus.py. Questions: evals/reranker-es-legal/questions-2026-11-15.json,\n54 hand-written Spanish questions in three types (literal, paraphrase, hard), each with gold\npassages. Nothing in the dataset was generated by a local model.\n\nProtocol (the llm-evaluation skill: deterministic retrieval metrics, counts before percentages):\n  * chunks: one per unit, prefixed with its article or annex heading; a unit longer than\n    --chunk-chars is cut at line boundaries (then at word boundaries), no overlap;\n  * dense: the embedding model through Ollama /api/embed, cosine on normalised vectors held in\n    memory with numpy (no vector database: 800-odd vectors do not need one);\n  * bm25: Okapi BM25 (k1 1.5, b 0.75) on lower-cased, accent-stripped word tokens; no stemming and\n    no stop-word list, so it is a plain lexical baseline;\n  * fusion: reciprocal rank fusion (k 60) of the dense and BM25 top --pool lists;\n  * rerank: the top N candidates of a first stage are re-scored by the cross-encoder through\n    /v1/rerank and sorted by that score; N is each value of --depths for dense, --pool for the rest;\n  * a question is a hit at k when any gold chunk is in the first k results; MRR@10 uses the first\n    gold chunk; every table carries counts, a Wilson 95 % interval, and the split by question type;\n  * paired comparison against dense: questions whose first gold chunk moved up, down or stayed,\n    and an exact two-sided sign test on the moves;\n  * latency, one request at a time: each query is embedded --reps times and reranked --reps times\n    at each depth after a warm-up; we keep each question's median, then report the median and the\n    95th percentile over the questions. The GPU's other tenants at run time are recorded.\n\nUsage:\n    python3 scripts/bench/reranker_es_legal.py --out /tmp/reranker-es.json\n",
  "script_sha256": "de73b4efbb454d86d8073c3842bf821b5a4c34c51313b3bb2543c72bc6756ec7",
  "corpus_file": "evals/reranker-es-legal/corpus-aiact-es-2024-1689.json",
  "corpus_source": "Reglamento (UE) 2024/1689, texto en español del Diario Oficial (DO L, 12.7.2024), CELEX 32024R1689, Oficina de Publicaciones de la UE",
  "corpus_source_sha256": "0ef11d381226cde823d0bbf03ed195ec98645b9a0b7e06b1668c7f46751aad8c",
  "corpus_file_sha256": "6c78567148f7cd509899488534ecc8ac76d1a79155647a10e54dd797996a3df4",
  "questions_file": "evals/reranker-es-legal/questions-2026-11-15.json",
  "questions_file_sha256": "00d8f00c34580bfec178871733df55fc0108fecf97dda3f2ae2665dc23ae7f48",
  "n_docs": 126,
  "n_units": 667,
  "n_chunks": 761,
  "chunk_chars_max_setting": 1200,
  "chunk_chars_median": 481,
  "chunk_chars_longest": 1337,
  "n_questions": 54,
  "questions_by_type": {
   "literal": 18,
   "paraphrase": 18,
   "hard": 18
  },
  "gold_chunks_per_question": {
   "1": 40,
   "2": 12,
   "5": 1,
   "3": 1
  },
  "settings": {
   "pool": 50,
   "depths": [
    10,
    25,
    50
   ],
   "reps": 5,
   "rrf_k": 60,
   "bm25": {
    "k1": 1.5,
    "b": 0.75
   },
   "ks": [
    1,
    3,
    5,
    10
   ]
  },
  "host": {
   "run_host": "spark-43d5",
   "gpu": "NVIDIA GB10, 580.173.02",
   "kernel": "6.17.0-1032-nvidia",
   "arch": "aarch64",
   "ollama_url": "http://localhost:11434",
   "ollama_version": "0.30.10",
   "embed_model": {
    "name": "bge-m3:latest",
    "digest": "7907646426070047a77226ac3e684fbbe8410524f7b4a74d02837e43f2146bab",
    "file_bytes": 1157672605,
    "details": {
     "parent_model": "",
     "format": "gguf",
     "family": "bert",
     "families": [
      "bert"
     ],
     "parameter_size": "566.70M",
     "quantization_level": "F16",
     "context_length": 8192,
     "embedding_length": 1024
    }
   },
   "reranker_url": "http://localhost:8201",
   "reranker_server": "vLLM 0.19.0+6bc3197f.nv26.04.48680843",
   "reranker_model": {
    "served_as": "reranker",
    "root": "BAAI/bge-reranker-v2-m3",
    "max_model_len": 8192
   },
   "ollama_resident_models": [
    {
     "name": "nomic-embed-text:latest",
     "size_bytes": 323150151
    },
    {
     "name": "bge-m3:latest",
     "size_bytes": 664000265
    },
    {
     "name": "llama3.1:8b",
     "size_bytes": 9205983477
    }
   ],
   "numpy": "2.5.0"
  },
  "gpu_before": {
   "utilization_pct_samples": [
    0,
    0,
    0,
    0,
    0
   ],
   "compute_apps": [
    {
     "process": "gnome-remote-desktop-daemon",
     "used": "176 MiB"
    },
    {
     "process": "python",
     "used": "351 MiB"
    },
    {
     "process": "llama-server",
     "used": "890 MiB"
    },
    {
     "process": "python3",
     "used": "1102 MiB"
    },
    {
     "process": "llama-server",
     "used": "543 MiB"
    },
    {
     "process": "VLLM::EngineCore",
     "used": "1876 MiB"
    },
    {
     "process": "llama-server",
     "used": "882 MiB"
    },
    {
     "process": "llama-server",
     "used": "507 MiB"
    },
    {
     "process": "llama-server",
     "used": "8993 MiB"
    }
   ]
  },
  "indexing": {
   "chunks": 761,
   "seconds": 11.84,
   "dim": 1024,
   "chunks_per_second": 64.2,
   "batch": 32
  },
  "methods": [
   {
    "method": "dense",
    "summary": {
     "n": 54,
     "hit@1": 41,
     "hit@1_ci95": [
      0.6305,
      0.8536
     ],
     "hit@3": 46,
     "hit@3_ci95": [
      0.734,
      0.923
     ],
     "hit@5": 47,
     "hit@5_ci95": [
      0.7558,
      0.9358
     ],
     "hit@10": 48,
     "hit@10_ci95": [
      0.7781,
      0.9481
     ],
     "mrr@10": 0.8057
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 8,
      "hit@1_ci95": [
       0.2456,
       0.6628
      ],
      "hit@3": 12,
      "hit@3_ci95": [
       0.4375,
       0.8372
      ],
      "hit@5": 13,
      "hit@5_ci95": [
       0.4913,
       0.875
      ],
      "hit@10": 14,
      "hit@10_ci95": [
       0.5478,
       0.91
      ],
      "mrr@10": 0.5561
     },
     "literal": {
      "n": 18,
      "hit@1": 17,
      "hit@1_ci95": [
       0.7424,
       0.9901
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 0.9722
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 16,
      "hit@1_ci95": [
       0.672,
       0.969
      ],
      "hit@3": 16,
      "hit@3_ci95": [
       0.672,
       0.969
      ],
      "hit@5": 16,
      "hit@5_ci95": [
       0.672,
       0.969
      ],
      "hit@10": 16,
      "hit@10_ci95": [
       0.672,
       0.969
      ],
      "mrr@10": 0.8889
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     1,
     1,
     1,
     14,
     1,
     1,
     1,
     12,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     3,
     1,
     1,
     41,
     13,
     3,
     null,
     2,
     1,
     1,
     1,
     2,
     5,
     null,
     1,
     1,
     7,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_100.par_3#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_26.par_10#2"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "anx_I#2"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_111.par_3#0"
     ],
     [
      "art_3.pt_4#0",
      "art_26.par_2#0",
      "art_26.par_5#1"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "art_3.pt_63#0"
     ],
     [
      "art_22.par_1#0",
      "art_23.par_1#0",
      "art_25.par_3#0"
     ],
     [
      "art_4#0",
      "art_26.par_5#0",
      "art_50.par_2#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_3#0",
      "art_73.par_6#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_1#0",
      "art_6.par_3#1"
     ],
     [
      "art_26.par_7#0",
      "art_26.par_5#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_6.par_1#0",
      "art_15.par_5#0"
     ],
     [
      "art_18.par_1#0",
      "art_22.par_3#1",
      "art_18.par_2#0"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_2#0",
      "art_27.par_1#1"
     ],
     [
      "art_48.par_3#0",
      "art_16#1",
      "art_48.par_5#0"
     ],
     [
      "art_53.par_1#0",
      "art_55.par_1#0",
      "art_53.par_3#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_15#0",
      "art_57.par_11#0"
     ],
     [
      "art_5.par_1#0",
      "art_5.par_1#1",
      "art_5.par_1#3"
     ],
     [
      "art_99.par_5#0",
      "art_99.par_4#0",
      "art_91.par_4#0"
     ],
     [
      "art_50.par_1#0",
      "art_5.par_4#0",
      "art_50.par_3#0"
     ],
     [
      "art_50.par_4#0",
      "art_3.pt_60#0",
      "art_50.par_2#0"
     ],
     [
      "art_99.par_6#0",
      "art_99.par_4#0",
      "art_99.par_5#0"
     ],
     [
      "art_2.par_1#0",
      "art_59.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "art_53.par_2#0",
      "art_54.par_6#0",
      "art_53.par_1#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_6.par_1#0"
     ],
     [
      "art_86.par_1#0",
      "art_75.par_3#0",
      "art_82.par_1#0"
     ],
     [
      "art_5.par_3#0",
      "art_5.par_2#0",
      "art_3.pt_36#0"
     ],
     [
      "art_73.par_4#0",
      "art_73.par_2#0",
      "art_73.par_6#0"
     ],
     [
      "art_55.par_1#0",
      "art_51.par_2#0",
      "art_3.pt_63#0"
     ],
     [
      "art_2.par_3#0",
      "art_2.par_6#0",
      "art_2.par_4#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_85#0",
      "art_89.par_2#0",
      "art_87#0"
     ],
     [
      "art_10.par_5#0",
      "art_10.par_2#1",
      "art_10.par_5#1"
     ],
     [
      "art_23.par_1#0",
      "art_24.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "art_17.par_4#0",
      "anx_VII#2"
     ],
     [
      "art_50.par_2#0",
      "art_50.par_4#1",
      "art_50.par_4#0"
     ],
     [
      "art_50.par_3#0",
      "art_46.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "anx_III.pt_4#0",
      "art_5.par_1#2",
      "art_3.pt_40#0"
     ],
     [
      "art_5.par_1#1",
      "art_3.pt_60#0",
      "art_3.pt_13#0"
     ],
     [
      "art_5.par_3#1",
      "art_50.par_3#0",
      "art_5.par_5#0"
     ],
     [
      "art_5.par_1#4",
      "art_26.par_10#0",
      "art_5.par_2#1"
     ],
     [
      "art_73.par_6#0",
      "art_20.par_2#0",
      "art_26.par_5#0"
     ],
     [
      "art_75.par_1#0",
      "art_74.par_6#0",
      "art_74.par_10#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_25.par_2#0"
     ],
     [
      "art_5.par_1#4",
      "art_5.par_2#1",
      "art_5.par_6#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_6#1",
      "art_5.par_1#3"
     ],
     [
      "art_111.par_2#0",
      "art_2.par_8#0",
      "art_111.par_1#0"
     ],
     [
      "art_58.par_3#0",
      "art_62.par_1#0",
      "art_58.par_2#1"
     ],
     [
      "art_15.par_4#0",
      "art_60.par_7#0",
      "art_46.par_2#0"
     ],
     [
      "art_79.par_2#1",
      "art_99.par_3#0",
      "art_2.par_11#0"
     ],
     [
      "art_18.par_1#0",
      "art_23.par_5#0",
      "art_44.par_2#0"
     ],
     [
      "anx_III.pt_5#1",
      "anx_III.pt_5#0",
      "anx_III.pt_2#0"
     ],
     [
      "art_43.par_4#0",
      "art_60.par_1#0",
      "art_6.par_3#1"
     ],
     [
      "art_15.par_5#0",
      "art_59.par_2#0",
      "art_10.par_3#0"
     ]
    ]
   },
   {
    "method": "bm25",
    "summary": {
     "n": 54,
     "hit@1": 23,
     "hit@1_ci95": [
      0.3033,
      0.5584
     ],
     "hit@3": 28,
     "hit@3_ci95": [
      0.3885,
      0.6461
     ],
     "hit@5": 31,
     "hit@5_ci95": [
      0.4416,
      0.6967
     ],
     "hit@10": 37,
     "hit@10_ci95": [
      0.5526,
      0.7932
     ],
     "mrr@10": 0.4948
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 2,
      "hit@1_ci95": [
       0.031,
       0.328
      ],
      "hit@3": 3,
      "hit@3_ci95": [
       0.0584,
       0.3922
      ],
      "hit@5": 4,
      "hit@5_ci95": [
       0.09,
       0.4522
      ],
      "hit@10": 7,
      "hit@10_ci95": [
       0.203,
       0.6138
      ],
      "mrr@10": 0.1782
     },
     "literal": {
      "n": 18,
      "hit@1": 14,
      "hit@1_ci95": [
       0.5478,
       0.91
      ],
      "hit@3": 15,
      "hit@3_ci95": [
       0.6078,
       0.9416
      ],
      "hit@5": 16,
      "hit@5_ci95": [
       0.672,
       0.969
      ],
      "hit@10": 16,
      "hit@10_ci95": [
       0.672,
       0.969
      ],
      "mrr@10": 0.8194
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 7,
      "hit@1_ci95": [
       0.203,
       0.6138
      ],
      "hit@3": 10,
      "hit@3_ci95": [
       0.3372,
       0.7544
      ],
      "hit@5": 11,
      "hit@5_ci95": [
       0.3862,
       0.797
      ],
      "hit@10": 14,
      "hit@10_ci95": [
       0.5478,
       0.91
      ],
      "mrr@10": 0.4869
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     14,
     1,
     1,
     1,
     1,
     4,
     1,
     1,
     1,
     1,
     1,
     12,
     1,
     2,
     20,
     1,
     5,
     3,
     null,
     3,
     1,
     1,
     23,
     7,
     9,
     1,
     1,
     26,
     2,
     1,
     1,
     7,
     null,
     6,
     8,
     null,
     24,
     null,
     null,
     4,
     29,
     1,
     2,
     null,
     15,
     null,
     null,
     1,
     22,
     6
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_81.par_1#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_20.par_1#0",
      "art_50.par_4#0"
     ],
     [
      "art_113#0",
      "art_73.par_8#0",
      "art_33.par_4#0"
     ],
     [
      "anx_IV.pt_1#1",
      "art_76.par_3#0",
      "art_3.pt_15#0"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "art_3.pt_64#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_26.par_1#0",
      "art_3.pt_56#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_4#0",
      "art_73.par_3#0"
     ],
     [
      "art_6.par_4#0",
      "art_80.par_3#0",
      "art_80.par_1#0"
     ],
     [
      "art_26.par_7#0",
      "art_49.par_3#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_9.par_6#0",
      "art_26.par_7#0"
     ],
     [
      "art_18.par_1#0",
      "art_47.par_1#0",
      "art_18.par_2#0"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_4#0",
      "anx_VIII.pt_4_3#0"
     ],
     [
      "art_48.par_3#0",
      "art_48.par_4#0",
      "art_48.par_5#0"
     ],
     [
      "art_53.par_3#0",
      "art_53.par_4#0",
      "art_55.par_2#0"
     ],
     [
      "art_57.par_1#0",
      "art_58.par_2#2",
      "art_59.par_1#2"
     ],
     [
      "art_5.par_1#0",
      "art_5.par_1#1",
      "art_5.par_5#1"
     ],
     [
      "art_26.par_5#0",
      "art_36.par_4#0",
      "art_22.par_4#0"
     ],
     [
      "art_50.par_1#0",
      "art_5.par_1#0",
      "art_52.par_3#0"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#1",
      "art_26.par_10#1"
     ],
     [
      "art_11.par_1#1",
      "art_34.par_2#0",
      "art_99.par_6#0"
     ],
     [
      "anx_VII#1",
      "art_3.pt_27#0",
      "art_41.par_4#0"
     ],
     [
      "anx_XI.pt_1#0",
      "anx_XI.intro#0",
      "art_53.par_2#0"
     ],
     [
      "art_25.par_1#0",
      "art_3.pt_4#0",
      "art_16#0"
     ],
     [
      "art_86.par_1#0",
      "art_5.par_3#1",
      "art_5.par_1#0"
     ],
     [
      "anx_III.pt_1#0",
      "art_3.pt_35#0",
      "art_3.pt_42#0"
     ],
     [
      "art_73.par_6#0",
      "art_83.par_1#0",
      "art_73.par_2#0"
     ],
     [
      "art_112.par_6#0",
      "art_88.par_1#0",
      "art_57.par_2#0"
     ],
     [
      "art_2.par_3#0",
      "art_3.pt_29#0",
      "art_3.pt_30#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_99.par_3#0",
      "art_65.par_1#0",
      "art_99.par_5#0"
     ],
     [
      "art_72.par_2#0",
      "art_10.par_5#1",
      "art_5.par_4#0"
     ],
     [
      "art_23.par_1#0",
      "art_23.par_5#0",
      "art_23.par_7#0"
     ],
     [
      "art_63.par_1#0",
      "anx_VII#2",
      "art_9.par_4#0"
     ],
     [
      "art_50.par_4#0",
      "art_50.par_7#0",
      "art_16#0"
     ],
     [
      "art_99.par_3#0",
      "art_99.par_5#0",
      "art_48.par_2#0"
     ],
     [
      "anx_IV.pt_1#0",
      "art_67.par_2#0",
      "art_19.par_2#0"
     ],
     [
      "art_3.pt_30#0",
      "art_3.pt_60#0",
      "art_5.par_1#0"
     ],
     [
      "art_52.par_5#0",
      "art_5.par_1#4",
      "art_74.par_7#0"
     ],
     [
      "art_3.pt_13#0",
      "art_99.par_5#0",
      "art_3.pt_60#0"
     ],
     [
      "art_93.par_3#0",
      "anx_VII#1",
      "art_27.par_5#0"
     ],
     [
      "art_14.par_1#0",
      "anx_VIII.pt_6#0",
      "art_67.par_2#0"
     ],
     [
      "art_43.par_3#1",
      "art_25.par_3#0",
      "art_3.pt_8#0"
     ],
     [
      "art_3.pt_66#0",
      "anx_III.pt_5#1",
      "art_62.par_3#0"
     ],
     [
      "art_5.par_1#2",
      "art_36.par_8#1",
      "art_3.pt_35#0"
     ],
     [
      "art_41.par_4#0",
      "art_111.par_3#0",
      "art_99.par_9#0"
     ],
     [
      "art_3.pt_14#0",
      "art_100.par_4#0",
      "art_93.par_3#0"
     ],
     [
      "art_23.par_2#0",
      "art_36.par_8#1",
      "art_52.par_1#0"
     ],
     [
      "art_14.par_2#0",
      "art_99.par_3#0",
      "art_24.par_2#0"
     ],
     [
      "art_3.pt_12#0",
      "art_29.par_2#0",
      "art_80.par_1#0"
     ],
     [
      "anx_III.pt_5#1",
      "art_112.par_1#0",
      "art_112.par_11#0"
     ],
     [
      "art_3.pt_67#0",
      "art_19.par_1#0",
      "art_19.par_2#0"
     ],
     [
      "art_93.par_3#0",
      "anx_XIII#0",
      "anx_XII.pt_2#0"
     ]
    ]
   },
   {
    "method": "rrf(dense,bm25)",
    "summary": {
     "n": 54,
     "hit@1": 34,
     "hit@1_ci95": [
      0.4963,
      0.7458
     ],
     "hit@3": 43,
     "hit@3_ci95": [
      0.671,
      0.8823
     ],
     "hit@5": 44,
     "hit@5_ci95": [
      0.6916,
      0.8962
     ],
     "hit@10": 48,
     "hit@10_ci95": [
      0.7781,
      0.9481
     ],
     "mrr@10": 0.7151
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 8,
      "hit@1_ci95": [
       0.2456,
       0.6628
      ],
      "hit@3": 10,
      "hit@3_ci95": [
       0.3372,
       0.7544
      ],
      "hit@5": 11,
      "hit@5_ci95": [
       0.3862,
       0.797
      ],
      "hit@10": 14,
      "hit@10_ci95": [
       0.5478,
       0.91
      ],
      "mrr@10": 0.525
     },
     "literal": {
      "n": 18,
      "hit@1": 15,
      "hit@1_ci95": [
       0.6078,
       0.9416
      ],
      "hit@3": 17,
      "hit@3_ci95": [
       0.7424,
       0.9901
      ],
      "hit@5": 17,
      "hit@5_ci95": [
       0.7424,
       0.9901
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 0.8981
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 11,
      "hit@1_ci95": [
       0.3862,
       0.797
      ],
      "hit@3": 16,
      "hit@3_ci95": [
       0.672,
       0.969
      ],
      "hit@5": 16,
      "hit@5_ci95": [
       0.672,
       0.969
      ],
      "hit@10": 16,
      "hit@10_ci95": [
       0.672,
       0.969
      ],
      "mrr@10": 0.7222
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     2,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     6,
     1,
     2,
     2,
     1,
     2,
     1,
     31,
     1,
     1,
     1,
     16,
     3,
     1,
     1,
     1,
     3,
     1,
     1,
     1,
     3,
     11,
     1,
     3,
     null,
     4,
     6,
     null,
     2,
     1,
     1,
     1,
     10,
     1,
     null,
     10,
     1,
     1,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_81.par_1#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_1.par_1#0"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_2.par_10#0"
     ],
     [
      "art_86.par_1#0",
      "art_3.pt_4#0",
      "art_26.par_4#0"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "art_3.pt_64#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_3.pt_56#0",
      "art_26.par_1#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_3#0",
      "art_73.par_4#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_1#0",
      "art_6.par_4#0"
     ],
     [
      "art_26.par_7#0",
      "art_26.par_5#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "anx_III.intro#0",
      "art_9.par_6#0"
     ],
     [
      "art_18.par_1#0",
      "art_22.par_3#1",
      "art_18.par_2#0"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_4#0",
      "art_27.par_2#0"
     ],
     [
      "art_48.par_3#0",
      "art_48.par_5#0",
      "art_16#1"
     ],
     [
      "art_53.par_3#0",
      "art_53.par_4#0",
      "art_55.par_2#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_11#0",
      "art_58.par_2#2"
     ],
     [
      "art_5.par_1#0",
      "art_5.par_1#1",
      "art_5.par_5#1"
     ],
     [
      "art_36.par_4#0",
      "art_99.par_5#0",
      "art_101.par_1#0"
     ],
     [
      "art_50.par_1#0",
      "art_50.par_4#1",
      "art_73.par_4#0"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_50.par_4#1"
     ],
     [
      "art_99.par_6#0",
      "art_99.par_9#0",
      "art_62.par_2#0"
     ],
     [
      "art_59.par_3#0",
      "art_59.par_2#0",
      "art_59.par_1#2"
     ],
     [
      "art_53.par_2#0",
      "anx_XI.pt_1#0",
      "art_54.par_6#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_16#0"
     ],
     [
      "art_86.par_1#0",
      "art_26.par_10#0",
      "art_14.par_4#0"
     ],
     [
      "art_3.pt_35#0",
      "art_5.par_2#0",
      "anx_III.pt_1#0"
     ],
     [
      "art_73.par_6#0",
      "art_73.par_2#0",
      "art_73.par_4#0"
     ],
     [
      "art_55.par_1#0",
      "art_53.par_1#0",
      "art_55.par_2#0"
     ],
     [
      "art_2.par_3#0",
      "art_78.par_2#0",
      "anx_III.pt_1#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_99.par_5#0",
      "art_99.par_4#0",
      "art_85#0"
     ],
     [
      "art_10.par_5#1",
      "art_10.par_5#0",
      "art_10.par_2#1"
     ],
     [
      "art_23.par_1#0",
      "art_23.par_4#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "anx_VII#2",
      "anx_VII#0"
     ],
     [
      "art_50.par_4#0",
      "art_50.par_7#0",
      "art_50.par_2#0"
     ],
     [
      "art_50.par_1#0",
      "art_2.par_1#0",
      "art_50.par_4#0"
     ],
     [
      "anx_III.pt_4#0",
      "anx_XI.pt_2#2",
      "anx_III.pt_7#0"
     ],
     [
      "art_3.pt_60#0",
      "art_5.par_1#2",
      "art_5.par_1#1"
     ],
     [
      "art_50.par_3#0",
      "art_5.par_3#0",
      "art_50.par_1#0"
     ],
     [
      "art_26.par_10#0",
      "art_5.par_1#4",
      "art_3.pt_60#0"
     ],
     [
      "art_52.par_1#0",
      "art_73.par_6#0",
      "art_93.par_3#0"
     ],
     [
      "art_14.par_1#0",
      "art_75.par_1#0",
      "art_57.par_10#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_43.par_3#1"
     ],
     [
      "art_5.par_2#0",
      "anx_III.pt_5#1",
      "anx_III.pt_7#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_1#0",
      "anx_III.pt_6#1"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_1#0",
      "art_111.par_2#0"
     ],
     [
      "art_24.par_1#0",
      "art_23.par_1#0",
      "art_57.par_5#0"
     ],
     [
      "art_14.par_4#1",
      "art_75.par_3#0",
      "art_3.pt_57#0"
     ],
     [
      "art_99.par_3#0",
      "art_86.par_1#0",
      "art_50.par_1#0"
     ],
     [
      "anx_VII#2",
      "art_19.par_1#0",
      "art_6.par_4#0"
     ],
     [
      "anx_III.pt_5#1",
      "art_46.par_2#0",
      "art_13.par_3#0"
     ],
     [
      "anx_III.pt_3#0",
      "art_15.par_4#0",
      "art_7.par_2#2"
     ],
     [
      "art_15.par_5#0",
      "anx_XI.pt_2#1",
      "art_10.par_1#0"
     ]
    ]
   },
   {
    "method": "dense top 10 -> rerank",
    "summary": {
     "n": 54,
     "hit@1": 41,
     "hit@1_ci95": [
      0.6305,
      0.8536
     ],
     "hit@3": 47,
     "hit@3_ci95": [
      0.7558,
      0.9358
     ],
     "hit@5": 48,
     "hit@5_ci95": [
      0.7781,
      0.9481
     ],
     "hit@10": 48,
     "hit@10_ci95": [
      0.7781,
      0.9481
     ],
     "mrr@10": 0.8154
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 8,
      "hit@1_ci95": [
       0.2456,
       0.6628
      ],
      "hit@3": 13,
      "hit@3_ci95": [
       0.4913,
       0.875
      ],
      "hit@5": 14,
      "hit@5_ci95": [
       0.5478,
       0.91
      ],
      "hit@10": 14,
      "hit@10_ci95": [
       0.5478,
       0.91
      ],
      "mrr@10": 0.5852
     },
     "literal": {
      "n": 18,
      "hit@1": 18,
      "hit@1_ci95": [
       0.8241,
       1.0
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 1.0
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 15,
      "hit@1_ci95": [
       0.6078,
       0.9416
      ],
      "hit@3": 16,
      "hit@3_ci95": [
       0.672,
       0.969
      ],
      "hit@5": 16,
      "hit@5_ci95": [
       0.672,
       0.969
      ],
      "hit@10": 16,
      "hit@10_ci95": [
       0.672,
       0.969
      ],
      "mrr@10": 0.8611
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     null,
     1,
     1,
     1,
     null,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     null,
     null,
     2,
     null,
     2,
     1,
     1,
     2,
     3,
     1,
     null,
     1,
     1,
     5,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_100.par_3#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_1.par_1#0"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_3.pt_4#0",
      "art_26.par_10#2",
      "art_26.par_5#1"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "anx_XIII#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_62.par_3#0",
      "art_50.par_1#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_4#0",
      "art_73.par_3#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_3#0",
      "art_7.par_1#0"
     ],
     [
      "art_26.par_7#0",
      "art_22.par_1#0",
      "art_26.par_11#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_9.par_5#0",
      "art_6.par_3#0"
     ],
     [
      "art_18.par_1#0",
      "art_47.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_2#0",
      "art_26.par_9#0"
     ],
     [
      "art_48.par_3#0",
      "art_16#1",
      "art_48.par_2#0"
     ],
     [
      "art_53.par_1#0",
      "art_55.par_1#0",
      "art_53.par_4#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_16#0",
      "art_57.par_13#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "art_99.par_5#0",
      "art_91.par_4#0",
      "art_91.par_5#0"
     ],
     [
      "art_50.par_1#0",
      "art_50.par_3#0",
      "art_50.par_4#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_50.par_2#0"
     ],
     [
      "art_99.par_6#0",
      "art_99.par_1#0",
      "art_99.par_5#0"
     ],
     [
      "art_50.par_1#0",
      "art_5.par_1#2",
      "art_5.par_1#1"
     ],
     [
      "art_53.par_2#0",
      "anx_XI.pt_1#0",
      "art_54.par_6#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_48.par_3#0"
     ],
     [
      "art_86.par_1#0",
      "art_82.par_1#0",
      "anx_III.pt_8#0"
     ],
     [
      "art_5.par_2#0",
      "art_26.par_10#0",
      "art_5.par_3#1"
     ],
     [
      "art_73.par_4#0",
      "art_73.par_3#0",
      "art_73.par_2#0"
     ],
     [
      "art_55.par_1#0",
      "art_53.par_1#0",
      "art_53.par_2#0"
     ],
     [
      "art_2.par_3#0",
      "art_2.par_10#0",
      "art_2.par_4#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_1#0",
      "art_111.par_2#0"
     ],
     [
      "art_85#0",
      "art_89.par_2#0",
      "art_87#0"
     ],
     [
      "art_10.par_5#0",
      "art_10.par_5#1",
      "art_10.par_2#1"
     ],
     [
      "art_23.par_1#0",
      "art_24.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "art_17.par_1#0",
      "art_17.par_4#0"
     ],
     [
      "art_50.par_2#0",
      "art_50.par_4#1",
      "art_50.par_4#0"
     ],
     [
      "art_5.par_1#3",
      "art_50.par_3#0",
      "art_5.par_1#2"
     ],
     [
      "art_5.par_1#3",
      "anx_III.pt_4#0",
      "art_10.par_5#0"
     ],
     [
      "art_5.par_1#1",
      "art_3.pt_60#0",
      "art_5.par_1#2"
     ],
     [
      "art_5.par_5#0",
      "art_5.par_3#1",
      "art_5.par_1#2"
     ],
     [
      "art_5.par_5#0",
      "art_5.par_1#4",
      "art_5.par_2#1"
     ],
     [
      "art_73.par_10#0",
      "art_26.par_5#0",
      "art_52.par_1#0"
     ],
     [
      "art_75.par_1#0",
      "art_79.par_9#0",
      "art_74.par_8#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_25.par_2#0"
     ],
     [
      "art_5.par_1#4",
      "art_5.par_5#0",
      "art_5.par_3#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_6#1",
      "art_5.par_1#3"
     ],
     [
      "art_2.par_8#0",
      "art_111.par_1#0",
      "art_111.par_3#0"
     ],
     [
      "art_58.par_2#1",
      "art_58.par_3#0",
      "art_62.par_1#0"
     ],
     [
      "art_14.par_4#1",
      "art_15.par_4#0",
      "art_79.par_5#0"
     ],
     [
      "art_99.par_3#0",
      "art_2.par_11#0",
      "art_86.par_1#0"
     ],
     [
      "art_18.par_1#0",
      "art_23.par_5#0",
      "art_22.par_3#1"
     ],
     [
      "anx_III.pt_5#1",
      "art_46.par_2#0",
      "anx_III.pt_2#0"
     ],
     [
      "art_6.par_3#0",
      "art_6.par_3#1",
      "art_6.par_1#0"
     ],
     [
      "art_15.par_5#0",
      "anx_XI.pt_2#1",
      "art_10.par_1#0"
     ]
    ]
   },
   {
    "method": "dense top 25 -> rerank",
    "summary": {
     "n": 54,
     "hit@1": 44,
     "hit@1_ci95": [
      0.6916,
      0.8962
     ],
     "hit@3": 49,
     "hit@3_ci95": [
      0.8009,
      0.9598
     ],
     "hit@5": 50,
     "hit@5_ci95": [
      0.8245,
      0.9708
     ],
     "hit@10": 51,
     "hit@10_ci95": [
      0.8489,
      0.9809
     ],
     "mrr@10": 0.8619
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 9,
      "hit@1_ci95": [
       0.2903,
       0.7097
      ],
      "hit@3": 13,
      "hit@3_ci95": [
       0.4913,
       0.875
      ],
      "hit@5": 14,
      "hit@5_ci95": [
       0.5478,
       0.91
      ],
      "hit@10": 15,
      "hit@10_ci95": [
       0.6078,
       0.9416
      ],
      "mrr@10": 0.6134
     },
     "literal": {
      "n": 18,
      "hit@1": 18,
      "hit@1_ci95": [
       0.8241,
       1.0
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 1.0
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 17,
      "hit@1_ci95": [
       0.7424,
       0.9901
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 0.9722
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     3,
     1,
     null,
     1,
     4,
     null,
     2,
     1,
     1,
     2,
     3,
     1,
     null,
     1,
     1,
     8,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_100.par_3#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_1.par_1#0"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_3.pt_4#0",
      "art_26.par_10#2",
      "art_26.par_5#1"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "anx_XIII#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_62.par_3#0",
      "art_50.par_1#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_4#0",
      "art_73.par_3#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_3#0",
      "art_7.par_1#0"
     ],
     [
      "art_26.par_7#0",
      "art_13.par_3#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_9.par_6#0",
      "art_6.par_3#1"
     ],
     [
      "art_18.par_1#0",
      "art_47.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_2#0",
      "art_27.par_5#0"
     ],
     [
      "art_48.par_3#0",
      "art_16#1",
      "art_48.par_2#0"
     ],
     [
      "art_53.par_1#0",
      "art_55.par_1#0",
      "art_53.par_4#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_16#0",
      "art_57.par_13#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "art_99.par_5#0",
      "art_91.par_4#0",
      "art_91.par_5#0"
     ],
     [
      "art_50.par_1#0",
      "art_50.par_3#0",
      "art_50.par_4#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_50.par_7#0"
     ],
     [
      "art_99.par_6#0",
      "art_62.par_2#0",
      "art_99.par_1#0"
     ],
     [
      "art_2.par_10#0",
      "art_50.par_1#0",
      "art_50.par_3#0"
     ],
     [
      "art_53.par_2#0",
      "anx_XI.pt_1#0",
      "art_54.par_6#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_48.par_3#0"
     ],
     [
      "art_86.par_1#0",
      "art_82.par_1#0",
      "art_14.par_4#0"
     ],
     [
      "art_14.par_5#0",
      "art_5.par_2#0",
      "art_26.par_10#0"
     ],
     [
      "art_73.par_4#0",
      "art_73.par_3#0",
      "art_73.par_2#0"
     ],
     [
      "art_55.par_1#0",
      "art_53.par_1#0",
      "art_53.par_2#0"
     ],
     [
      "art_2.par_3#0",
      "art_2.par_10#0",
      "art_2.par_4#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_1#0",
      "art_111.par_2#0"
     ],
     [
      "art_85#0",
      "art_89.par_2#0",
      "art_87#0"
     ],
     [
      "art_10.par_5#0",
      "art_10.par_5#1",
      "art_10.par_2#1"
     ],
     [
      "art_23.par_1#0",
      "art_24.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "art_17.par_1#0",
      "art_17.par_4#0"
     ],
     [
      "art_50.par_2#0",
      "art_50.par_4#1",
      "art_50.par_4#0"
     ],
     [
      "art_5.par_1#3",
      "art_26.par_7#0",
      "art_2.par_11#0"
     ],
     [
      "art_6.par_5#0",
      "art_5.par_1#3",
      "anx_III.pt_4#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_3.pt_60#0"
     ],
     [
      "art_5.par_5#0",
      "art_5.par_3#1",
      "art_5.par_3#0"
     ],
     [
      "art_5.par_1#2",
      "art_5.par_3#0",
      "art_5.par_1#3"
     ],
     [
      "art_73.par_10#0",
      "art_90.par_1#0",
      "art_90.par_3#0"
     ],
     [
      "art_14.par_1#0",
      "art_75.par_1#0",
      "art_26.par_5#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_5.par_1#4",
      "art_5.par_5#0",
      "art_5.par_3#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_6#1",
      "art_5.par_1#3"
     ],
     [
      "art_2.par_8#0",
      "art_111.par_1#0",
      "art_111.par_3#0"
     ],
     [
      "art_58.par_2#1",
      "art_58.par_3#0",
      "art_62.par_1#0"
     ],
     [
      "art_14.par_4#1",
      "art_15.par_4#0",
      "art_9.par_5#0"
     ],
     [
      "art_99.par_3#0",
      "art_2.par_11#0",
      "art_86.par_1#0"
     ],
     [
      "art_18.par_1#0",
      "art_23.par_5#0",
      "art_22.par_3#1"
     ],
     [
      "anx_III.pt_5#1",
      "art_46.par_2#0",
      "anx_VIII.pt_12#0"
     ],
     [
      "art_6.par_3#0",
      "art_6.par_3#1",
      "art_12.par_1#0"
     ],
     [
      "art_15.par_5#0",
      "anx_XI.pt_2#1",
      "art_10.par_1#0"
     ]
    ]
   },
   {
    "method": "dense top 50 -> rerank",
    "summary": {
     "n": 54,
     "hit@1": 44,
     "hit@1_ci95": [
      0.6916,
      0.8962
     ],
     "hit@3": 48,
     "hit@3_ci95": [
      0.7781,
      0.9481
     ],
     "hit@5": 49,
     "hit@5_ci95": [
      0.8009,
      0.9598
     ],
     "hit@10": 51,
     "hit@10_ci95": [
      0.8489,
      0.9809
     ],
     "mrr@10": 0.8588
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 9,
      "hit@1_ci95": [
       0.2903,
       0.7097
      ],
      "hit@3": 12,
      "hit@3_ci95": [
       0.4375,
       0.8372
      ],
      "hit@5": 13,
      "hit@5_ci95": [
       0.4913,
       0.875
      ],
      "hit@10": 15,
      "hit@10_ci95": [
       0.6078,
       0.9416
      ],
      "mrr@10": 0.6042
     },
     "literal": {
      "n": 18,
      "hit@1": 18,
      "hit@1_ci95": [
       0.8241,
       1.0
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 1.0
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 17,
      "hit@1_ci95": [
       0.7424,
       0.9901
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 0.9722
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     4,
     1,
     8,
     1,
     6,
     null,
     2,
     1,
     1,
     2,
     3,
     1,
     null,
     1,
     1,
     12,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_100.par_3#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_1.par_1#0"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_3.pt_4#0",
      "art_26.par_10#2",
      "art_26.par_5#1"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "anx_XIII#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_62.par_3#0",
      "art_50.par_1#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_4#0",
      "art_73.par_3#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_3#0",
      "art_7.par_1#0"
     ],
     [
      "art_26.par_7#0",
      "art_13.par_3#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_9.par_6#0",
      "art_6.par_3#1"
     ],
     [
      "art_18.par_1#0",
      "art_47.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_2#0",
      "art_27.par_3#0"
     ],
     [
      "art_48.par_3#0",
      "art_16#1",
      "art_48.par_2#0"
     ],
     [
      "art_53.par_1#0",
      "art_55.par_1#0",
      "art_53.par_4#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_16#0",
      "art_57.par_13#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "art_99.par_5#0",
      "art_91.par_4#0",
      "art_91.par_5#0"
     ],
     [
      "art_50.par_1#0",
      "art_50.par_3#0",
      "art_50.par_4#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_50.par_7#0"
     ],
     [
      "art_99.par_6#0",
      "art_62.par_2#0",
      "art_99.par_1#0"
     ],
     [
      "art_2.par_10#0",
      "art_50.par_1#0",
      "art_50.par_4#1"
     ],
     [
      "art_53.par_2#0",
      "anx_XI.pt_1#0",
      "art_54.par_6#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_48.par_3#0"
     ],
     [
      "art_86.par_1#0",
      "art_82.par_1#0",
      "art_14.par_4#0"
     ],
     [
      "art_14.par_5#0",
      "art_5.par_2#0",
      "art_26.par_10#0"
     ],
     [
      "art_73.par_4#0",
      "art_73.par_3#0",
      "art_73.par_2#0"
     ],
     [
      "art_55.par_1#0",
      "art_53.par_1#0",
      "art_53.par_1#1"
     ],
     [
      "art_2.par_3#0",
      "art_2.par_10#0",
      "art_2.par_4#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_1#0",
      "art_111.par_2#0"
     ],
     [
      "art_85#0",
      "art_89.par_2#0",
      "art_87#0"
     ],
     [
      "art_10.par_5#0",
      "art_10.par_5#1",
      "art_10.par_2#1"
     ],
     [
      "art_23.par_1#0",
      "art_24.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "art_17.par_1#0",
      "art_17.par_4#0"
     ],
     [
      "art_50.par_2#0",
      "art_50.par_4#1",
      "art_50.par_4#0"
     ],
     [
      "art_5.par_1#3",
      "art_26.par_7#0",
      "art_2.par_11#0"
     ],
     [
      "art_6.par_5#0",
      "art_3.pt_13#0",
      "art_5.par_1#3"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_3.pt_60#0"
     ],
     [
      "art_5.par_5#0",
      "art_5.par_1#1",
      "art_5.par_3#1"
     ],
     [
      "art_5.par_1#2",
      "art_5.par_3#0",
      "art_10.par_5#0"
     ],
     [
      "art_73.par_10#0",
      "art_90.par_1#0",
      "art_14.par_1#0"
     ],
     [
      "art_14.par_1#0",
      "art_75.par_1#0",
      "art_26.par_5#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_5.par_1#4",
      "art_5.par_5#0",
      "art_5.par_3#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_6#1",
      "art_5.par_1#3"
     ],
     [
      "art_2.par_8#0",
      "art_111.par_1#0",
      "art_111.par_3#0"
     ],
     [
      "art_58.par_2#1",
      "art_58.par_3#0",
      "art_62.par_1#0"
     ],
     [
      "art_14.par_4#1",
      "art_15.par_4#0",
      "art_9.par_5#0"
     ],
     [
      "art_99.par_3#0",
      "art_2.par_11#0",
      "art_86.par_1#0"
     ],
     [
      "art_18.par_1#0",
      "art_23.par_5#0",
      "art_22.par_3#1"
     ],
     [
      "anx_III.pt_5#1",
      "art_46.par_2#0",
      "art_6.par_5#0"
     ],
     [
      "art_6.par_3#0",
      "art_6.par_3#1",
      "art_12.par_1#0"
     ],
     [
      "art_15.par_5#0",
      "anx_XI.pt_2#1",
      "art_10.par_1#0"
     ]
    ]
   },
   {
    "method": "bm25 top 50 -> rerank",
    "summary": {
     "n": 54,
     "hit@1": 43,
     "hit@1_ci95": [
      0.671,
      0.8823
     ],
     "hit@3": 45,
     "hit@3_ci95": [
      0.7126,
      0.9098
     ],
     "hit@5": 45,
     "hit@5_ci95": [
      0.7126,
      0.9098
     ],
     "hit@10": 46,
     "hit@10_ci95": [
      0.734,
      0.923
     ],
     "mrr@10": 0.8179
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 9,
      "hit@1_ci95": [
       0.2903,
       0.7097
      ],
      "hit@3": 10,
      "hit@3_ci95": [
       0.3372,
       0.7544
      ],
      "hit@5": 10,
      "hit@5_ci95": [
       0.3372,
       0.7544
      ],
      "hit@10": 11,
      "hit@10_ci95": [
       0.3862,
       0.797
      ],
      "mrr@10": 0.537
     },
     "literal": {
      "n": 18,
      "hit@1": 18,
      "hit@1_ci95": [
       0.8241,
       1.0
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 1.0
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 16,
      "hit@1_ci95": [
       0.672,
       0.969
      ],
      "hit@3": 17,
      "hit@3_ci95": [
       0.7424,
       0.9901
      ],
      "hit@5": 17,
      "hit@5_ci95": [
       0.7424,
       0.9901
      ],
      "hit@10": 17,
      "hit@10_ci95": [
       0.7424,
       0.9901
      ],
      "mrr@10": 0.9167
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     null,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     null,
     1,
     1,
     null,
     1,
     null,
     null,
     2,
     1,
     1,
     1,
     null,
     1,
     null,
     null,
     1,
     6,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_5.par_1#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_1.par_1#0"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_94#0"
     ],
     [
      "art_3.pt_4#0",
      "art_26.par_10#2",
      "art_50.par_4#1"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "anx_XIII#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_62.par_3#0",
      "art_3.pt_56#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_4#0",
      "art_73.par_3#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_3#0",
      "art_7.par_1#0"
     ],
     [
      "art_26.par_7#0",
      "art_13.par_3#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_9.par_6#0",
      "art_26.par_7#0"
     ],
     [
      "art_18.par_1#0",
      "art_47.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_2#0",
      "art_27.par_3#0"
     ],
     [
      "art_48.par_3#0",
      "art_16#1",
      "art_48.par_2#0"
     ],
     [
      "art_53.par_1#0",
      "art_55.par_1#0",
      "art_53.par_4#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_16#0",
      "art_57.par_5#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "art_99.par_5#0",
      "art_101.par_1#0",
      "art_78.par_3#0"
     ],
     [
      "art_50.par_1#0",
      "art_5.par_1#0",
      "art_26.par_11#0"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_50.par_2#0"
     ],
     [
      "art_99.par_6#0",
      "art_62.par_2#0",
      "art_99.par_1#0"
     ],
     [
      "art_50.par_4#1",
      "art_50.par_3#0",
      "art_5.par_1#1"
     ],
     [
      "art_53.par_2#0",
      "anx_XI.pt_1#0",
      "art_54.par_6#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_16#0"
     ],
     [
      "art_86.par_1#0",
      "art_82.par_1#0",
      "art_14.par_4#0"
     ],
     [
      "art_14.par_5#0",
      "art_5.par_2#0",
      "art_26.par_10#0"
     ],
     [
      "art_73.par_4#0",
      "art_73.par_3#0",
      "art_73.par_2#0"
     ],
     [
      "art_55.par_1#0",
      "art_53.par_1#0",
      "art_53.par_2#0"
     ],
     [
      "art_2.par_3#0",
      "art_75.par_2#0",
      "anx_III.pt_8#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_1#0",
      "art_111.par_2#0"
     ],
     [
      "art_85#0",
      "art_89.par_2#0",
      "art_99.par_4#0"
     ],
     [
      "art_10.par_5#0",
      "art_10.par_5#1",
      "art_10.par_2#1"
     ],
     [
      "art_23.par_1#0",
      "art_24.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "art_17.par_1#0",
      "art_17.par_4#0"
     ],
     [
      "art_50.par_2#0",
      "art_50.par_4#1",
      "art_50.par_4#0"
     ],
     [
      "art_50.par_1#0",
      "art_10.par_5#1",
      "art_2.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_14.par_4#0",
      "art_7.par_2#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_3.pt_60#0"
     ],
     [
      "art_5.par_1#0",
      "art_91.par_5#0",
      "art_5.par_1#1"
     ],
     [
      "art_5.par_1#2",
      "art_5.par_1#1",
      "art_59.par_1#0"
     ],
     [
      "art_52.par_1#0",
      "anx_III.pt_8#0",
      "art_15.par_5#0"
     ],
     [
      "art_14.par_1#0",
      "art_75.par_1#0",
      "art_14.par_4#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_5.par_2#0",
      "art_13.par_3#0",
      "anx_III.pt_7#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_6#1",
      "anx_III.pt_1#0"
     ],
     [
      "art_111.par_1#0",
      "art_111.par_3#0",
      "art_111.par_2#0"
     ],
     [
      "art_2.par_8#0",
      "art_57.par_5#0",
      "art_8.par_2#0"
     ],
     [
      "art_14.par_4#1",
      "art_23.par_2#0",
      "art_5.par_3#0"
     ],
     [
      "art_99.par_3#0",
      "art_86.par_1#0",
      "art_50.par_1#0"
     ],
     [
      "art_24.par_5#0",
      "art_11.par_1#1",
      "anx_VI#0"
     ],
     [
      "anx_III.pt_5#1",
      "art_46.par_2#0",
      "art_5.par_3#0"
     ],
     [
      "art_19.par_1#0",
      "art_14.par_4#0",
      "art_14.par_4#1"
     ],
     [
      "art_15.par_5#0",
      "anx_XI.pt_2#1",
      "anx_XII.pt_2#0"
     ]
    ]
   },
   {
    "method": "rrf(dense,bm25) top 50 -> rerank",
    "summary": {
     "n": 54,
     "hit@1": 44,
     "hit@1_ci95": [
      0.6916,
      0.8962
     ],
     "hit@3": 49,
     "hit@3_ci95": [
      0.8009,
      0.9598
     ],
     "hit@5": 50,
     "hit@5_ci95": [
      0.8245,
      0.9708
     ],
     "hit@10": 50,
     "hit@10_ci95": [
      0.8245,
      0.9708
     ],
     "mrr@10": 0.8596
    },
    "by_type": {
     "hard": {
      "n": 18,
      "hit@1": 9,
      "hit@1_ci95": [
       0.2903,
       0.7097
      ],
      "hit@3": 13,
      "hit@3_ci95": [
       0.4913,
       0.875
      ],
      "hit@5": 14,
      "hit@5_ci95": [
       0.5478,
       0.91
      ],
      "hit@10": 14,
      "hit@10_ci95": [
       0.5478,
       0.91
      ],
      "mrr@10": 0.6065
     },
     "literal": {
      "n": 18,
      "hit@1": 18,
      "hit@1_ci95": [
       0.8241,
       1.0
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 1.0
     },
     "paraphrase": {
      "n": 18,
      "hit@1": 17,
      "hit@1_ci95": [
       0.7424,
       0.9901
      ],
      "hit@3": 18,
      "hit@3_ci95": [
       0.8241,
       1.0
      ],
      "hit@5": 18,
      "hit@5_ci95": [
       0.8241,
       1.0
      ],
      "hit@10": 18,
      "hit@10_ci95": [
       0.8241,
       1.0
      ],
      "mrr@10": 0.9722
     }
    },
    "first_gold_rank": [
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     2,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     1,
     3,
     1,
     null,
     1,
     4,
     null,
     2,
     1,
     1,
     2,
     3,
     1,
     null,
     1,
     1,
     12,
     1
    ],
    "top3": [
     [
      "art_99.par_3#0",
      "art_100.par_2#0",
      "art_100.par_3#0"
     ],
     [
      "art_26.par_6#0",
      "art_19.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_1.par_1#0"
     ],
     [
      "art_113#0",
      "art_111.par_2#0",
      "art_111.par_1#0"
     ],
     [
      "art_3.pt_4#0",
      "art_26.par_10#2",
      "art_26.par_5#1"
     ],
     [
      "art_51.par_2#0",
      "art_51.par_1#0",
      "anx_XIII#0"
     ],
     [
      "art_22.par_1#0",
      "art_54.par_1#0",
      "art_24.par_1#0"
     ],
     [
      "art_4#0",
      "art_62.par_3#0",
      "art_50.par_1#0"
     ],
     [
      "art_73.par_2#0",
      "art_73.par_4#0",
      "art_73.par_3#0"
     ],
     [
      "art_6.par_3#0",
      "art_7.par_3#0",
      "art_7.par_1#0"
     ],
     [
      "art_26.par_7#0",
      "art_13.par_3#0",
      "art_22.par_1#0"
     ],
     [
      "anx_III.pt_4#0",
      "art_9.par_6#0",
      "art_6.par_3#1"
     ],
     [
      "art_18.par_1#0",
      "art_47.par_1#0",
      "art_22.par_3#1"
     ],
     [
      "art_27.par_1#0",
      "art_27.par_2#0",
      "art_27.par_3#0"
     ],
     [
      "art_48.par_3#0",
      "art_16#1",
      "art_48.par_2#0"
     ],
     [
      "art_53.par_1#0",
      "art_55.par_1#0",
      "art_53.par_4#0"
     ],
     [
      "art_57.par_1#0",
      "art_57.par_16#0",
      "art_57.par_13#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_5.par_1#3"
     ],
     [
      "art_99.par_5#0",
      "art_91.par_4#0",
      "art_91.par_5#0"
     ],
     [
      "art_50.par_1#0",
      "art_5.par_1#0",
      "art_50.par_3#0"
     ],
     [
      "art_3.pt_60#0",
      "art_50.par_4#0",
      "art_50.par_7#0"
     ],
     [
      "art_99.par_6#0",
      "art_62.par_2#0",
      "art_99.par_1#0"
     ],
     [
      "art_2.par_10#0",
      "art_50.par_1#0",
      "art_50.par_4#1"
     ],
     [
      "art_53.par_2#0",
      "anx_XI.pt_1#0",
      "art_54.par_6#0"
     ],
     [
      "art_25.par_1#0",
      "art_25.par_3#0",
      "art_48.par_3#0"
     ],
     [
      "art_86.par_1#0",
      "art_82.par_1#0",
      "art_14.par_4#0"
     ],
     [
      "art_14.par_5#0",
      "art_5.par_2#0",
      "art_26.par_10#0"
     ],
     [
      "art_73.par_4#0",
      "art_73.par_3#0",
      "art_73.par_2#0"
     ],
     [
      "art_55.par_1#0",
      "art_53.par_1#0",
      "art_53.par_1#1"
     ],
     [
      "art_2.par_3#0",
      "art_2.par_10#0",
      "art_2.par_4#0"
     ],
     [
      "art_111.par_3#0",
      "art_111.par_1#0",
      "art_111.par_2#0"
     ],
     [
      "art_85#0",
      "art_89.par_2#0",
      "art_87#0"
     ],
     [
      "art_10.par_5#0",
      "art_10.par_5#1",
      "art_10.par_2#1"
     ],
     [
      "art_23.par_1#0",
      "art_24.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_63.par_1#0",
      "art_17.par_1#0",
      "art_17.par_4#0"
     ],
     [
      "art_50.par_2#0",
      "art_50.par_4#1",
      "art_50.par_4#0"
     ],
     [
      "art_5.par_1#3",
      "art_26.par_7#0",
      "art_2.par_11#0"
     ],
     [
      "art_6.par_5#0",
      "art_5.par_1#3",
      "anx_III.pt_4#0"
     ],
     [
      "art_5.par_1#1",
      "art_5.par_1#0",
      "art_3.pt_60#0"
     ],
     [
      "art_5.par_5#0",
      "art_91.par_5#0",
      "art_5.par_1#1"
     ],
     [
      "art_5.par_1#2",
      "art_5.par_3#0",
      "art_5.par_1#3"
     ],
     [
      "art_73.par_10#0",
      "art_90.par_1#0",
      "art_90.par_3#0"
     ],
     [
      "art_14.par_1#0",
      "art_75.par_1#0",
      "art_26.par_5#0"
     ],
     [
      "art_25.par_3#0",
      "art_25.par_1#0",
      "art_23.par_2#0"
     ],
     [
      "art_5.par_1#4",
      "art_5.par_5#0",
      "art_5.par_3#0"
     ],
     [
      "art_5.par_1#2",
      "anx_III.pt_6#1",
      "art_5.par_1#3"
     ],
     [
      "art_2.par_8#0",
      "art_111.par_1#0",
      "art_111.par_3#0"
     ],
     [
      "art_58.par_2#1",
      "art_58.par_3#0",
      "art_62.par_1#0"
     ],
     [
      "art_14.par_4#1",
      "art_15.par_4#0",
      "art_82.par_1#0"
     ],
     [
      "art_99.par_3#0",
      "art_2.par_11#0",
      "art_86.par_1#0"
     ],
     [
      "art_18.par_1#0",
      "art_23.par_5#0",
      "art_22.par_3#1"
     ],
     [
      "anx_III.pt_5#1",
      "art_46.par_2#0",
      "art_5.par_3#0"
     ],
     [
      "art_19.par_1#0",
      "art_14.par_4#0",
      "art_6.par_3#0"
     ],
     [
      "art_15.par_5#0",
      "anx_XI.pt_2#1",
      "anx_XII.pt_2#0"
     ]
    ]
   }
  ],
  "paired_vs_dense": {
   "bm25": {
    "moved_up": 0,
    "moved_down": 28,
    "same": 26,
    "sign_test_p": 0.0,
    "top1_gained": 0,
    "top1_lost": 18,
    "top1_sign_test_p": 0.0
   },
   "rrf(dense,bm25)": {
    "moved_up": 3,
    "moved_down": 15,
    "same": 36,
    "sign_test_p": 0.0075,
    "top1_gained": 2,
    "top1_lost": 9,
    "top1_sign_test_p": 0.0654
   },
   "dense top 10 -> rerank": {
    "moved_up": 5,
    "moved_down": 8,
    "same": 41,
    "sign_test_p": 0.5811,
    "top1_gained": 3,
    "top1_lost": 3,
    "top1_sign_test_p": 1.0
   },
   "dense top 25 -> rerank": {
    "moved_up": 6,
    "moved_down": 7,
    "same": 41,
    "sign_test_p": 1.0,
    "top1_gained": 6,
    "top1_lost": 3,
    "top1_sign_test_p": 0.5078
   },
   "dense top 50 -> rerank": {
    "moved_up": 7,
    "moved_down": 6,
    "same": 41,
    "sign_test_p": 1.0,
    "top1_gained": 6,
    "top1_lost": 3,
    "top1_sign_test_p": 0.5078
   },
   "bm25 top 50 -> rerank": {
    "moved_up": 5,
    "moved_down": 7,
    "same": 42,
    "sign_test_p": 0.7744,
    "top1_gained": 4,
    "top1_lost": 2,
    "top1_sign_test_p": 0.6875
   },
   "rrf(dense,bm25) top 50 -> rerank": {
    "moved_up": 6,
    "moved_down": 7,
    "same": 41,
    "sign_test_p": 1.0,
    "top1_gained": 6,
    "top1_lost": 3,
    "top1_sign_test_p": 0.5078
   }
  },
  "gold_in_candidates": {
   "dense": 52,
   "bm25": 46,
   "rrf(dense,bm25)": 51
  },
  "latency": {
   "query_embedding": {
    "n_questions": 54,
    "median_ms": 175.4,
    "p95_ms": 197.6,
    "min_ms": 144.3,
    "max_ms": 209.5
   },
   "numpy_search": {
    "n_questions": 54,
    "median_ms": 9.0,
    "p95_ms": 29.9,
    "min_ms": 3.4,
    "max_ms": 67.9
   },
   "rerank": {
    "10": {
     "n_questions": 54,
     "median_ms": 42.1,
     "p95_ms": 61.5,
     "min_ms": 28.4,
     "max_ms": 66.6
    },
    "25": {
     "n_questions": 54,
     "median_ms": 87.2,
     "p95_ms": 120.5,
     "min_ms": 57.7,
     "max_ms": 137.7
    },
    "50": {
     "n_questions": 54,
     "median_ms": 159.2,
     "p95_ms": 175.0,
     "min_ms": 126.1,
     "max_ms": 203.4
    },
    "50 (bm25 candidates)": {
     "n_questions": 54,
     "median_ms": 148.4,
     "p95_ms": 177.5,
     "min_ms": 112.2,
     "max_ms": 191.1
    },
    "50 (rrf(dense,bm25) candidates)": {
     "n_questions": 54,
     "median_ms": 161.2,
     "p95_ms": 186.5,
     "min_ms": 125.4,
     "max_ms": 193.5
    }
   }
  },
  "questions_with_unstable_rerank_order": {
   "dense top 10": 0,
   "dense top 25": 0,
   "dense top 50": 0,
   "bm25 top 50": 0,
   "rrf(dense,bm25) top 50": 0
  },
  "question_ids": [
   "l01",
   "l02",
   "l03",
   "l04",
   "l05",
   "l06",
   "l07",
   "l08",
   "l09",
   "l10",
   "l11",
   "l12",
   "l13",
   "l14",
   "l15",
   "l16",
   "l17",
   "l18",
   "p01",
   "p02",
   "p03",
   "p04",
   "p05",
   "p06",
   "p07",
   "p08",
   "p09",
   "p10",
   "p11",
   "p12",
   "p13",
   "p14",
   "p15",
   "p16",
   "p17",
   "p18",
   "h01",
   "h02",
   "h03",
   "h04",
   "h05",
   "h06",
   "h07",
   "h08",
   "h09",
   "h10",
   "h11",
   "h12",
   "h13",
   "h14",
   "h15",
   "h16",
   "h17",
   "h18"
  ],
  "question_types": [
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "literal",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "paraphrase",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard",
   "hard"
  ],
  "gpu_after": {
   "utilization_pct_samples": [
    39,
    39,
    0,
    0,
    0
   ],
   "compute_apps": [
    {
     "process": "gnome-remote-desktop-daemon",
     "used": "176 MiB"
    },
    {
     "process": "python",
     "used": "351 MiB"
    },
    {
     "process": "llama-server",
     "used": "890 MiB"
    },
    {
     "process": "python3",
     "used": "1102 MiB"
    },
    {
     "process": "llama-server",
     "used": "543 MiB"
    },
    {
     "process": "VLLM::EngineCore",
     "used": "1876 MiB"
    },
    {
     "process": "llama-server",
     "used": "882 MiB"
    },
    {
     "process": "llama-server",
     "used": "507 MiB"
    },
    {
     "process": "llama-server",
     "used": "8993 MiB"
    }
   ]
  },
  "finished": "2026-11-16T05:59:39"
 }
}