examples/trec_29_deep_documents_cache_impact_depth_20.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTerrier 0.8.0 has loaded Terrier 5.6 (built by craigmacdonald on 2021-09-17 13:27)\n",
      "\n",
      "No etc/terrier.properties, using terrier.default.properties for bootstrap configuration.\n"
     ]
    }
   ],
   "source": [
    "# Start/initialize PyTerrier.\n",
    "from pyterrier import started, init\n",
    "\n",
    "if not started():\n",
    "    init(tqdm=\"auto\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "edition = 29\n",
    "track = \"deep.documents\"\n",
    "dataset_name = \"msmarco-document/trec-dl-2020/judged\"\n",
    "contents_field = \"body\"\n",
    "depth = 20"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "from pyterrier.datasets import get_dataset\n",
    "from ir_datasets import load\n",
    "\n",
    "dataset = get_dataset(f\"irds:{dataset_name}\")\n",
    "ir_dataset = load(dataset_name)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "from tempfile import TemporaryDirectory\n",
    "from pathlib import Path\n",
    "\n",
    "tmp_dir = TemporaryDirectory()\n",
    "cache_dir = Path(tmp_dir.name)\n",
    "index_dir = Path(\"cache\") / \"indices\" / dataset_name.split(\"/\")[0]\n",
    "result_dir = Path(\n",
    "    \"/mnt/ceph/storage/data-in-progress/data-research/\"\n",
    "    \"web-search/web-search-trec/trec-system-runs\"\n",
    ") / f\"trec{edition}\" / track\n",
    "result_files = list(result_dir.iterdir())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "from pyterrier.index import IterDictIndexer\n",
    "\n",
    "if not index_dir.exists():\n",
    "    indexer = IterDictIndexer(str(index_dir.absolute()))\n",
    "    indexer.index(\n",
    "        dataset.get_corpus_iter(),\n",
    "        fields=[contents_field]\n",
    "    )"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "Load results:   0%|          | 0/64 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "b46d9ae707f540bf8b9d98f8186afa52"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyterrier.io import read_results\n",
    "from pyterrier import Transformer\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "results = [\n",
    "    Transformer.from_df(read_results(result_file))\n",
    "    for result_file in tqdm(result_files, desc=\"Load results\")\n",
    "]\n",
    "results_names = [result_file.stem.replace(\"input.\", \"\") for result_file in result_files]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "from ir_axioms.axiom import ArgUC, STMC1_f\n",
    "\n",
    "axioms = [\n",
    "    # ~ArgUC(),  # Very slow due to network access.\n",
    "    ~STMC1_f(),  # Rather slow due many similarity calculations.\n",
    "]\n",
    "axiom_names = [axiom.axiom.name for axiom in axioms]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "Computing system axiomatic preferences:   0%|          | 0/64 [00:00<?, ?system/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "868736a342434460bb3a159412c9412d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Computing query axiom preferences:   0%|          | 0/45 [00:00<?, ?query/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "65424427d9f44f0ebc7cf5da6fba3697"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Computing query axiom preferences:   0%|          | 0/45 [00:00<?, ?query/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "5ad09b7179244f1bba5ca464c930b458"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Computing query axiom preferences:   0%|          | 0/45 [00:00<?, ?query/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "3cd59f5e44f0411dbbe9d0a17ff4a00f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": "Computing query axiom preferences:   0%|          | 0/45 [00:00<?, ?query/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "144ba5042222427ebabdf7ae1432aad3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Input \u001B[0;32mIn [8]\u001B[0m, in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m      1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;21;01mir_axioms\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mbackend\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mpyterrier\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mexperiment\u001B[39;00m \u001B[38;5;28;01mimport\u001B[39;00m AxiomaticExperiment\n\u001B[0;32m----> 3\u001B[0m \u001B[43mAxiomaticExperiment\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m      4\u001B[0m \u001B[43m    \u001B[49m\u001B[43mretrieval_systems\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mresults\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      5\u001B[0m \u001B[43m    \u001B[49m\u001B[43mtopics\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdataset\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_topics\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      6\u001B[0m \u001B[43m    \u001B[49m\u001B[43mqrels\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdataset\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mget_qrels\u001B[49m\u001B[43m(\u001B[49m\u001B[43m)\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      7\u001B[0m \u001B[43m    \u001B[49m\u001B[43mindex\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mindex_dir\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      8\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdataset\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mir_dataset\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m      9\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcontents_accessor\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcontents_field\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     10\u001B[0m \u001B[43m    \u001B[49m\u001B[43maxioms\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxioms\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     11\u001B[0m \u001B[43m    \u001B[49m\u001B[43maxiom_names\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43maxiom_names\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     12\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdepth\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mdepth\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     13\u001B[0m \u001B[43m    \u001B[49m\u001B[43mfilter_by_qrels\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m     14\u001B[0m \u001B[43m    \u001B[49m\u001B[43mfilter_by_topics\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mFalse\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m     15\u001B[0m \u001B[43m    \u001B[49m\u001B[43mverbose\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;28;43;01mTrue\u001B[39;49;00m\u001B[43m,\u001B[49m\n\u001B[1;32m     16\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcache_dir\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mcache_dir\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     17\u001B[0m \u001B[43m)\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreferences\u001B[49m\n",
      "File \u001B[0;32m/usr/lib/python3.9/functools.py:993\u001B[0m, in \u001B[0;36mcached_property.__get__\u001B[0;34m(self, instance, owner)\u001B[0m\n\u001B[1;32m    991\u001B[0m val \u001B[38;5;241m=\u001B[39m cache\u001B[38;5;241m.\u001B[39mget(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mattrname, _NOT_FOUND)\n\u001B[1;32m    992\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m val \u001B[38;5;129;01mis\u001B[39;00m _NOT_FOUND:\n\u001B[0;32m--> 993\u001B[0m     val \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[43minstance\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    994\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[1;32m    995\u001B[0m         cache[\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mattrname] \u001B[38;5;241m=\u001B[39m val\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/experiment.py:142\u001B[0m, in \u001B[0;36mAxiomaticExperiment.preferences\u001B[0;34m(self)\u001B[0m\n\u001B[1;32m    136\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose:\n\u001B[1;32m    137\u001B[0m     pipelines \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    138\u001B[0m         pipelines,\n\u001B[1;32m    139\u001B[0m         desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing system axiomatic preferences\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    140\u001B[0m         unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msystem\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    141\u001B[0m     )\n\u001B[0;32m--> 142\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m concat([\n\u001B[1;32m    143\u001B[0m     pipeline\u001B[38;5;241m.\u001B[39mtransform(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtopics)\n\u001B[1;32m    144\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m pipeline \u001B[38;5;129;01min\u001B[39;00m pipelines\n\u001B[1;32m    145\u001B[0m ])\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/experiment.py:143\u001B[0m, in \u001B[0;36m<listcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m    136\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose:\n\u001B[1;32m    137\u001B[0m     pipelines \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    138\u001B[0m         pipelines,\n\u001B[1;32m    139\u001B[0m         desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing system axiomatic preferences\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    140\u001B[0m         unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124msystem\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    141\u001B[0m     )\n\u001B[1;32m    142\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m concat([\n\u001B[0;32m--> 143\u001B[0m     \u001B[43mpipeline\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtopics\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    144\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m pipeline \u001B[38;5;129;01min\u001B[39;00m pipelines\n\u001B[1;32m    145\u001B[0m ])\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pyterrier/transformer.py:879\u001B[0m, in \u001B[0;36mComposedPipeline.transform\u001B[0;34m(self, topics)\u001B[0m\n\u001B[1;32m    877\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mtransform\u001B[39m(\u001B[38;5;28mself\u001B[39m, topics):\n\u001B[1;32m    878\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmodels:\n\u001B[0;32m--> 879\u001B[0m         topics \u001B[38;5;241m=\u001B[39m \u001B[43mm\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtopics\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    880\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m topics\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:60\u001B[0m, in \u001B[0;36mPerGroupTransformer.transform\u001B[0;34m(self, topics_or_res)\u001B[0m\n\u001B[1;32m     54\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose:\n\u001B[1;32m     55\u001B[0m     \u001B[38;5;66;03m# Show progress during reranking queries.\u001B[39;00m\n\u001B[1;32m     56\u001B[0m     tqdm\u001B[38;5;241m.\u001B[39mpandas(\n\u001B[1;32m     57\u001B[0m         desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mdescription,\n\u001B[1;32m     58\u001B[0m         unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39munit,\n\u001B[1;32m     59\u001B[0m     )\n\u001B[0;32m---> 60\u001B[0m     query_rankings \u001B[38;5;241m=\u001B[39m \u001B[43mquery_rankings\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mprogress_apply\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     61\u001B[0m \u001B[43m        \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform_group\u001B[49m\n\u001B[1;32m     62\u001B[0m \u001B[43m    \u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     63\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m     64\u001B[0m     query_rankings \u001B[38;5;241m=\u001B[39m query_rankings\u001B[38;5;241m.\u001B[39mapply(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtransform_group)\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/tqdm/std.py:814\u001B[0m, in \u001B[0;36mtqdm.pandas.<locals>.inner_generator.<locals>.inner\u001B[0;34m(df, func, *args, **kwargs)\u001B[0m\n\u001B[1;32m    811\u001B[0m \u001B[38;5;66;03m# Apply the provided function (in **kwargs)\u001B[39;00m\n\u001B[1;32m    812\u001B[0m \u001B[38;5;66;03m# on the df using our wrapper (which provides bar updating)\u001B[39;00m\n\u001B[1;32m    813\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m--> 814\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mgetattr\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mdf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdf_function\u001B[49m\u001B[43m)\u001B[49m\u001B[43m(\u001B[49m\u001B[43mwrapper\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    815\u001B[0m \u001B[38;5;28;01mfinally\u001B[39;00m:\n\u001B[1;32m    816\u001B[0m     t\u001B[38;5;241m.\u001B[39mclose()\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pandas/core/groupby/groupby.py:1414\u001B[0m, in \u001B[0;36mGroupBy.apply\u001B[0;34m(self, func, *args, **kwargs)\u001B[0m\n\u001B[1;32m   1412\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m option_context(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mmode.chained_assignment\u001B[39m\u001B[38;5;124m\"\u001B[39m, \u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m   1413\u001B[0m     \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m-> 1414\u001B[0m         result \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_python_apply_general\u001B[49m\u001B[43m(\u001B[49m\u001B[43mf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_selected_obj\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1415\u001B[0m     \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m:\n\u001B[1;32m   1416\u001B[0m         \u001B[38;5;66;03m# gh-20949\u001B[39;00m\n\u001B[1;32m   1417\u001B[0m         \u001B[38;5;66;03m# try again, with .apply acting as a filtering\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1421\u001B[0m         \u001B[38;5;66;03m# fails on *some* columns, e.g. a numeric operation\u001B[39;00m\n\u001B[1;32m   1422\u001B[0m         \u001B[38;5;66;03m# on a string grouper column\u001B[39;00m\n\u001B[1;32m   1424\u001B[0m         \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_group_selection_context():\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pandas/core/groupby/groupby.py:1455\u001B[0m, in \u001B[0;36mGroupBy._python_apply_general\u001B[0;34m(self, f, data, not_indexed_same)\u001B[0m\n\u001B[1;32m   1429\u001B[0m \u001B[38;5;129m@final\u001B[39m\n\u001B[1;32m   1430\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_python_apply_general\u001B[39m(\n\u001B[1;32m   1431\u001B[0m     \u001B[38;5;28mself\u001B[39m,\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1434\u001B[0m     not_indexed_same: \u001B[38;5;28mbool\u001B[39m \u001B[38;5;241m|\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m,\n\u001B[1;32m   1435\u001B[0m ) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m DataFrame \u001B[38;5;241m|\u001B[39m Series:\n\u001B[1;32m   1436\u001B[0m     \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m   1437\u001B[0m \u001B[38;5;124;03m    Apply function f in python space\u001B[39;00m\n\u001B[1;32m   1438\u001B[0m \n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m   1453\u001B[0m \u001B[38;5;124;03m        data after applying f\u001B[39;00m\n\u001B[1;32m   1454\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m-> 1455\u001B[0m     values, mutated \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgrouper\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mapply\u001B[49m\u001B[43m(\u001B[49m\u001B[43mf\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdata\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43maxis\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1457\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m not_indexed_same \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m   1458\u001B[0m         not_indexed_same \u001B[38;5;241m=\u001B[39m mutated \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mmutated\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/pandas/core/groupby/ops.py:761\u001B[0m, in \u001B[0;36mBaseGrouper.apply\u001B[0;34m(self, f, data, axis)\u001B[0m\n\u001B[1;32m    759\u001B[0m \u001B[38;5;66;03m# group might be modified\u001B[39;00m\n\u001B[1;32m    760\u001B[0m group_axes \u001B[38;5;241m=\u001B[39m group\u001B[38;5;241m.\u001B[39maxes\n\u001B[0;32m--> 761\u001B[0m res \u001B[38;5;241m=\u001B[39m \u001B[43mf\u001B[49m\u001B[43m(\u001B[49m\u001B[43mgroup\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    762\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m mutated \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m _is_indexed_like(res, group_axes, axis):\n\u001B[1;32m    763\u001B[0m     mutated \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/venv/lib/python3.9/site-packages/tqdm/std.py:809\u001B[0m, in \u001B[0;36mtqdm.pandas.<locals>.inner_generator.<locals>.inner.<locals>.wrapper\u001B[0;34m(*args, **kwargs)\u001B[0m\n\u001B[1;32m    803\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwrapper\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[1;32m    804\u001B[0m     \u001B[38;5;66;03m# update tbar correctly\u001B[39;00m\n\u001B[1;32m    805\u001B[0m     \u001B[38;5;66;03m# it seems `pandas apply` calls `func` twice\u001B[39;00m\n\u001B[1;32m    806\u001B[0m     \u001B[38;5;66;03m# on the first column/row to decide whether it can\u001B[39;00m\n\u001B[1;32m    807\u001B[0m     \u001B[38;5;66;03m# take a fast or slow code path; so stop when t.total==t.n\u001B[39;00m\n\u001B[1;32m    808\u001B[0m     t\u001B[38;5;241m.\u001B[39mupdate(n\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1\u001B[39m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m t\u001B[38;5;241m.\u001B[39mtotal \u001B[38;5;129;01mor\u001B[39;00m t\u001B[38;5;241m.\u001B[39mn \u001B[38;5;241m<\u001B[39m t\u001B[38;5;241m.\u001B[39mtotal \u001B[38;5;28;01melse\u001B[39;00m \u001B[38;5;241m0\u001B[39m)\n\u001B[0;32m--> 809\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mfunc\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:107\u001B[0m, in \u001B[0;36mAxiomTransformer.transform_group\u001B[0;34m(self, topics_or_res)\u001B[0m\n\u001B[1;32m    104\u001B[0m \u001B[38;5;66;03m# Load document list.\u001B[39;00m\n\u001B[1;32m    105\u001B[0m documents \u001B[38;5;241m=\u001B[39m load_documents(topics_or_res)\n\u001B[0;32m--> 107\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtransform_query_ranking\u001B[49m\u001B[43m(\u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocuments\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mtopics_or_res\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:308\u001B[0m, in \u001B[0;36mAxiomaticPreferences.transform_query_ranking\u001B[0;34m(self, query, documents, results)\u001B[0m\n\u001B[1;32m    301\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;241m0\u001B[39m \u001B[38;5;241m<\u001B[39m logger\u001B[38;5;241m.\u001B[39mlevel \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m DEBUG:\n\u001B[1;32m    302\u001B[0m         \u001B[38;5;66;03m# Very verbose progress bars.\u001B[39;00m\n\u001B[1;32m    303\u001B[0m         document_pairs \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    304\u001B[0m             document_pairs,\n\u001B[1;32m    305\u001B[0m             desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing axiom preference\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    306\u001B[0m             unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpair\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    307\u001B[0m         )\n\u001B[0;32m--> 308\u001B[0m     results_pairs[column] \u001B[38;5;241m=\u001B[39m [\n\u001B[1;32m    309\u001B[0m         axiom\u001B[38;5;241m.\u001B[39mpreference(context, query, document1, document2)\n\u001B[1;32m    310\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m document1, document2 \u001B[38;5;129;01min\u001B[39;00m document_pairs\n\u001B[1;32m    311\u001B[0m     ]\n\u001B[1;32m    313\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m results_pairs\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/backend/pyterrier/transformers.py:309\u001B[0m, in \u001B[0;36m<listcomp>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m    301\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose \u001B[38;5;129;01mand\u001B[39;00m \u001B[38;5;241m0\u001B[39m \u001B[38;5;241m<\u001B[39m logger\u001B[38;5;241m.\u001B[39mlevel \u001B[38;5;241m<\u001B[39m\u001B[38;5;241m=\u001B[39m DEBUG:\n\u001B[1;32m    302\u001B[0m         \u001B[38;5;66;03m# Very verbose progress bars.\u001B[39;00m\n\u001B[1;32m    303\u001B[0m         document_pairs \u001B[38;5;241m=\u001B[39m tqdm(\n\u001B[1;32m    304\u001B[0m             document_pairs,\n\u001B[1;32m    305\u001B[0m             desc\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mComputing axiom preference\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    306\u001B[0m             unit\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mpair\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[1;32m    307\u001B[0m         )\n\u001B[1;32m    308\u001B[0m     results_pairs[column] \u001B[38;5;241m=\u001B[39m [\n\u001B[0;32m--> 309\u001B[0m         \u001B[43maxiom\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreference\u001B[49m\u001B[43m(\u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocument1\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mdocument2\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    310\u001B[0m         \u001B[38;5;28;01mfor\u001B[39;00m document1, document2 \u001B[38;5;129;01min\u001B[39;00m document_pairs\n\u001B[1;32m    311\u001B[0m     ]\n\u001B[1;32m    313\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m results_pairs\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/cache.py:52\u001B[0m, in \u001B[0;36mCachedAxiom.preference\u001B[0;34m(self, context, query, document1, document2)\u001B[0m\n\u001B[1;32m     49\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m preference\n\u001B[1;32m     51\u001B[0m \u001B[38;5;66;03m# Cache miss.\u001B[39;00m\n\u001B[0;32m---> 52\u001B[0m preference \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43maxiom\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mpreference\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     53\u001B[0m \u001B[43m    \u001B[49m\u001B[43mcontext\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     54\u001B[0m \u001B[43m    \u001B[49m\u001B[43mquery\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     55\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdocument1\u001B[49m\u001B[43m,\u001B[49m\n\u001B[1;32m     56\u001B[0m \u001B[43m    \u001B[49m\u001B[43mdocument2\u001B[49m\n\u001B[1;32m     57\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     58\u001B[0m cache[key] \u001B[38;5;241m=\u001B[39m preference\n\u001B[1;32m     59\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m preference\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/axiom/term_similarity.py:30\u001B[0m, in \u001B[0;36m_STMC1.preference\u001B[0;34m(self, context, query, document1, document2)\u001B[0m\n\u001B[1;32m     25\u001B[0m document2_terms \u001B[38;5;241m=\u001B[39m context\u001B[38;5;241m.\u001B[39mterm_set(document2)\n\u001B[1;32m     26\u001B[0m query_terms \u001B[38;5;241m=\u001B[39m context\u001B[38;5;241m.\u001B[39mterm_set(query)\n\u001B[1;32m     28\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m strictly_greater(\n\u001B[1;32m     29\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39maverage_similarity(document1_terms, query_terms),\n\u001B[0;32m---> 30\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43maverage_similarity\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdocument2_terms\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mquery_terms\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     31\u001B[0m )\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/modules/similarity.py:73\u001B[0m, in \u001B[0;36mTermSimilarityMixin.average_similarity\u001B[0;34m(self, terms1, terms2)\u001B[0m\n\u001B[1;32m     71\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(terms1) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(terms2) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m     72\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;241m0\u001B[39m\n\u001B[0;32m---> 73\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[43mmean\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m     74\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msimilarity\u001B[49m\u001B[43m(\u001B[49m\u001B[43mterm1\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mterm2\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     75\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mterm1\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mterms1\u001B[49m\n\u001B[1;32m     76\u001B[0m \u001B[43m    \u001B[49m\u001B[38;5;28;43;01mfor\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mterm2\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;129;43;01min\u001B[39;49;00m\u001B[43m \u001B[49m\u001B[43mterms2\u001B[49m\n\u001B[1;32m     77\u001B[0m \u001B[43m\u001B[49m\u001B[43m)\u001B[49m\n",
      "File \u001B[0;32m/usr/lib/python3.9/statistics.py:312\u001B[0m, in \u001B[0;36mmean\u001B[0;34m(data)\u001B[0m\n\u001B[1;32m    296\u001B[0m \u001B[38;5;124;03m\"\"\"Return the sample arithmetic mean of data.\u001B[39;00m\n\u001B[1;32m    297\u001B[0m \n\u001B[1;32m    298\u001B[0m \u001B[38;5;124;03m>>> mean([1, 2, 3, 4, 4])\u001B[39;00m\n\u001B[0;32m   (...)\u001B[0m\n\u001B[1;32m    309\u001B[0m \u001B[38;5;124;03mIf ``data`` is empty, StatisticsError will be raised.\u001B[39;00m\n\u001B[1;32m    310\u001B[0m \u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[1;32m    311\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28miter\u001B[39m(data) \u001B[38;5;129;01mis\u001B[39;00m data:\n\u001B[0;32m--> 312\u001B[0m     data \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mlist\u001B[39;49m\u001B[43m(\u001B[49m\u001B[43mdata\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    313\u001B[0m n \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mlen\u001B[39m(data)\n\u001B[1;32m    314\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m n \u001B[38;5;241m<\u001B[39m \u001B[38;5;241m1\u001B[39m:\n",
      "File \u001B[0;32m~/Development/webis/ir_axioms/ir_axioms/modules/similarity.py:74\u001B[0m, in \u001B[0;36m<genexpr>\u001B[0;34m(.0)\u001B[0m\n\u001B[1;32m     71\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(terms1) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m \u001B[38;5;129;01mor\u001B[39;00m \u001B[38;5;28mlen\u001B[39m(terms2) \u001B[38;5;241m==\u001B[39m \u001B[38;5;241m0\u001B[39m:\n\u001B[1;32m     72\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;241m0\u001B[39m\n\u001B[1;32m     73\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m mean(\n\u001B[0;32m---> 74\u001B[0m     \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43msimilarity\u001B[49m(term1, term2)\n\u001B[1;32m     75\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m term1 \u001B[38;5;129;01min\u001B[39;00m terms1\n\u001B[1;32m     76\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m term2 \u001B[38;5;129;01min\u001B[39;00m terms2\n\u001B[1;32m     77\u001B[0m )\n",
      "\u001B[0;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "from ir_axioms.backend.pyterrier.experiment import AxiomaticExperiment\n",
    "\n",
    "AxiomaticExperiment(\n",
    "    retrieval_systems=results,\n",
    "    topics=dataset.get_topics(),\n",
    "    qrels=dataset.get_qrels(),\n",
    "    index=index_dir,\n",
    "    dataset=ir_dataset,\n",
    "    contents_accessor=contents_field,\n",
    "    axioms=axioms,\n",
    "    axiom_names=axiom_names,\n",
    "    depth=depth,\n",
    "    filter_by_qrels=False,\n",
    "    filter_by_topics=False,\n",
    "    verbose=True,\n",
    "    cache_dir=cache_dir,\n",
    ").preferences"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}