confounder_data_acces.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import re\n",
    "from typing import Dict, List\n",
    "from collections import Counter, defaultdict\n",
    "from itertools import combinations\n",
    "import pandas as pd\n",
    "import json\n",
    "import numpy as np"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "You can find the full dataset with annotated inter-object causality at https://drive.google.com/file/d/17CTPMoZ4uJH6cSQaxD6Vmyv_bVLJwVJp/view\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Some auxiliary functions"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [],
   "source": [
    "def clean_up(x: str) -> str:  # to deal with e.g. '(\\'outfield\\', \"pitcher\\'s mound\"), or 'None', or \"\"?\"\"\n",
    "    cleaned_up = re.sub(r'(?<=\\w)\\'(?=\\w)', '@@TMP@@', x) \\\n",
    "        .replace('\\'', '\\\"') \\\n",
    "        .replace('\\\\\\'', '\\\"') \\\n",
    "        .replace('\\\"\\\"?\\\"\\\"', '\\\"?\\\"') \\\n",
    "        .replace('(', '[') \\\n",
    "        .replace(')', ']') \\\n",
    "        .replace('@@TMP@@', '\\'') \\\n",
    "        .replace('None', 'null')\n",
    "    return cleaned_up\n",
    "\n",
    "def max_response(row, w_for_cfdnce_lvl: Dict[str, List[int]]):\n",
    "    '''\n",
    "\n",
    "    Returns: most frequent response + its frequency as a fraction of maximum frequency\n",
    "\n",
    "    '''\n",
    "    c = Counter()\n",
    "    for k, cfdnce_lvl in zip(row['cause_directions'], row['confidences']):\n",
    "        c.update({k: w_for_cfdnce_lvl[cfdnce_lvl]})\n",
    "    max_resp, count = sorted(c.items(), key=lambda item: item[1], reverse=True)[0]\n",
    "    count_fraction = count / len(row['cause_directions'])\n",
    "    return max_resp, count_fraction, count"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          pair_of_objects                             cause_directions                 confounders                                                             confidences      word_X    word_Y\n",
      "0         trick // skater     [y-to-x, y-to-x, y-to-x, y-to-x, y-to-x]                  [, , , , ]  [confidence_3, confidence_3, confidence_3, confidence_3, confidence_2]       trick    skater\n",
      "1         sinks // vanity    [x-is-y, y-to-x, x-to-y, z-to-xy, x-to-y]  [bathroom, , , bathroom, ]  [confidence_3, confidence_2, confidence_2, confidence_2, confidence_2]       sinks    vanity\n",
      "2  toiletries // products    [z-to-xy, y-to-x, y-to-x, y-to-x, x-to-y]                 [?, , , , ]  [confidence_2, confidence_2, confidence_2, confidence_2, confidence_3]  toiletries  products\n",
      "3             roof // sky  [z-to-xy, x-to-y, z-to-xy, x-to-y, z-to-xy]            [?, , ?, , rain]  [confidence_1, confidence_3, confidence_1, confidence_2, confidence_2]        roof       sky\n",
      "4       laptops // office     [y-to-x, y-to-x, y-to-x, y-to-x, y-to-x]                  [, , , , ]  [confidence_1, confidence_3, confidence_2, confidence_2, confidence_3]     laptops    office\n"
     ]
    }
   ],
   "source": [
    "DATA_PATH = 'mturk_responses_clean.csv'\n",
    "responses = pd.read_csv(DATA_PATH,\n",
    "                        converters={col: lambda x: json.loads(clean_up(x))\n",
    "                                    for col in ['cause_directions', 'confounders', 'confidences']},\n",
    "                        index_col=0\n",
    "                        )\n",
    "responses[['word_X', 'word_Y']] = pd.DataFrame([p.split(\" // \") for p in responses['pair_of_objects'].tolist()],\n",
    "                                               index=responses.index)\n",
    "print(responses.head())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Filter the responses to use only those with sufficient agreement, possibly weighting agreement by confidence level"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     word_X  word_Y max_resp\n",
      "0     trick  skater   y-to-x\n",
      "4   laptops  office   y-to-x\n",
      "7    person   shirt   x-to-y\n",
      "8     table     man  z-to-xy\n",
      "10     face    tree  z-to-xy\n"
     ]
    }
   ],
   "source": [
    "w_for_cfdnce_setting = {\n",
    "    \"no_cfdnce_weight\": [1, 1, 1],\n",
    "    \"half_cfdnce_weight\": [.5, .75, 1],\n",
    "    \"full_cfdnce_weight\": [1 / 3, 2 / 3, 3 / 3],\n",
    "}\n",
    "min_agreement = .8\n",
    "cfdnce_setting = \"no_cfdnce_weight\"\n",
    "cfdnce_weighting = w_for_cfdnce_setting[cfdnce_setting]\n",
    "w_for_cfdnce_lvl = {\n",
    "    \"confidence_1\": cfdnce_weighting[0],\n",
    "    \"confidence_2\": cfdnce_weighting[1],\n",
    "    \"confidence_3\": cfdnce_weighting[2],\n",
    "    None: 0, # Ignore some cases where input is None\n",
    "}\n",
    "responses[['max_resp', 'max_resp_fraction', 'max_resp_count']] = pd.DataFrame(\n",
    "    [max_response(row, w_for_cfdnce_lvl) for _, row in responses.iterrows()], index=responses.index)\n",
    "filtered_responses = \\\n",
    "    responses[responses.apply(lambda row:\n",
    "                              row['max_resp_fraction'] >= min_agreement and\n",
    "                              row['max_resp'] != 'x-is-y' and\n",
    "                              len(row['cause_directions']) >= 5,\n",
    "                                                                   axis=1)]\n",
    "gt_for_pair = filtered_responses[['word_X', 'word_Y', 'max_resp']]\n",
    "print(gt_for_pair.head())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "As explained in the paper, for DeVLBERT only one effect variable is used (in other words, the confounders are effectively causes).\n",
    "To extract triplets from the data, use the following code."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "778\n",
      "['laptops⬅office➡computers', 'laptops⬅office➡telephone', 'laptops⬅office➡office chair', 'laptops⬅office➡calculator', 'laptops⬅office➡monitors', 'computers⬅office➡telephone', 'computers⬅office➡office chair', 'computers⬅office➡calculator', 'computers⬅office➡monitors', 'telephone⬅office➡office chair']\n"
     ]
    }
   ],
   "source": [
    "cause_only_responses = filtered_responses[filtered_responses.apply(lambda row:\n",
    "                                                                   row['max_resp'] in ['x-to-y', 'y-to-x'],\n",
    "                                                                   axis=1)]\n",
    "x_to_y = [(a.word_X, a.word_Y) if a.max_resp == 'x-to-y' else (a.word_Y, a.word_X) for a in\n",
    "          cause_only_responses.itertuples(index=False)]\n",
    "ys_for_x = defaultdict(list)\n",
    "for x, y in x_to_y:\n",
    "    ys_for_x[x] += [y]\n",
    "\n",
    "\n",
    "confnd_triples = [f\"{eff1}⬅{cause}➡{eff2}\" for cause in ys_for_x for eff1, eff2 in\n",
    "                  combinations(ys_for_x[cause], 2) if\n",
    "                  len(ys_for_x[cause]) > 1]\n",
    "print(len(confnd_triples))\n",
    "print(confnd_triples[:10])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "source": [
    "pd.options.mode.chained_assignment = None  # default='warn'\n",
    "class GT:\n",
    "\n",
    "    def __init__(self, gt_for_pair):\n",
    "        self.cache = {}\n",
    "        self.gt_for_pair = gt_for_pair\n",
    "\n",
    "    def get_known_gts(self, effect_object_id):\n",
    "        if effect_object_id in self.cache:\n",
    "            return self.cache[effect_object_id]\n",
    "        else:\n",
    "            known_gts = self.gt_for_pair[\n",
    "                (effect_object_id == self.gt_for_pair['ID_X']) | (effect_object_id == self.gt_for_pair['ID_Y'])]\n",
    "            known_gts['cause_candidate'] = known_gts['ID_X'].where(known_gts['ID_Y'] == effect_object_id,\n",
    "                                                                   known_gts['ID_Y'])\n",
    "            known_gts['cause_candidate_label'] = np.where(\n",
    "                (known_gts['ID_X'] == known_gts['cause_candidate']) & (known_gts['max_resp'] == 'x-to-y'),\n",
    "                'cause', np.where(known_gts['max_resp'] == 'z-to-xy', 'mere_correlate', 'effect'))\n",
    "            self.cache[effect_object_id] = known_gts\n",
    "            return known_gts\n",
    "\n",
    "with open(f\"DeVLBert/dic/objects_vocab.txt\", \"r\") as vocab:\n",
    "    CLASSES = ['background'] + [line.strip() for line in vocab]\n",
    "def word_to_id(word: str):\n",
    "    return CLASSES.index(word) if word in CLASSES else None\n",
    "\n",
    "gt_for_pair['ID_X'] = gt_for_pair.apply(lambda row: word_to_id(row['word_X']), axis=1)\n",
    "gt_for_pair['ID_Y'] = gt_for_pair.apply(lambda row: word_to_id(row['word_Y']), axis=1)\n",
    "gt = GT(gt_for_pair=gt_for_pair)\n",
    "pd.set_option('display.expand_frame_repr', False)\n",
    "print(gt.get_known_gts(effect_object_id=word_to_id('office')))\n",
    "print(gt.get_known_gts(effect_object_id=word_to_id('egg')))\n",
    "print(gt.get_known_gts(effect_object_id=word_to_id('shirt')))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": 51,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      word_X        word_Y max_resp  ID_X  ID_Y  cause_candidate cause_candidate_label\n",
      "4    laptops        office   y-to-x  1229  1081             1229                effect\n",
      "319   office     computers   x-to-y  1081  1032             1032                effect\n",
      "346   office     telephone   x-to-y  1081   582              582                effect\n",
      "366   office  office chair   x-to-y  1081   552              552                effect\n",
      "748   office    calculator   x-to-y  1081    13               13                effect\n",
      "765   office      monitors   x-to-y  1081   220              220                effect\n",
      "    word_X word_Y max_resp  ID_X  ID_Y  cause_candidate cause_candidate_label\n",
      "232    egg   yolk   x-to-y   525     1                1                effect\n",
      "          word_X  word_Y max_resp  ID_X  ID_Y  cause_candidate cause_candidate_label\n",
      "7         person   shirt   x-to-y   365    52              365                 cause\n",
      "11        sleeve   shirt   y-to-x  1497    52             1497                effect\n",
      "30          sock   shirt  z-to-xy   585    52              585        mere_correlate\n",
      "43         woman   shirt   x-to-y    91    52               91                 cause\n",
      "81      building   shirt  z-to-xy   178    52              178        mere_correlate\n",
      "118         eyes   shirt  z-to-xy   547    52              547        mere_correlate\n",
      "176         girl   shirt   x-to-y   140    52              140                 cause\n",
      "198        shirt   jeans  z-to-xy    52    48               48        mere_correlate\n",
      "215        watch   shirt  z-to-xy   336    52              336        mere_correlate\n",
      "229         ball   shirt  z-to-xy   568    52              568        mere_correlate\n",
      "236         tree   shirt  z-to-xy   292    52              292        mere_correlate\n",
      "254         wall   shirt  z-to-xy   249    52              249        mere_correlate\n",
      "277          boy   shirt   x-to-y   227    52              227                 cause\n",
      "282        water   shirt  z-to-xy   184    52              184        mere_correlate\n",
      "283        shirt  shorts  z-to-xy    52    45               45        mere_correlate\n",
      "301          sky   shirt  z-to-xy    73    52               73        mere_correlate\n",
      "309       window   shirt  z-to-xy   454    52              454        mere_correlate\n",
      "323         shoe   shirt  z-to-xy   243    52              243        mere_correlate\n",
      "328   sunglasses   shirt  z-to-xy   625    52              625        mere_correlate\n",
      "381        shoes   shirt  z-to-xy   771    52              771        mere_correlate\n",
      "394       jersey   shirt  z-to-xy   133    52              133        mere_correlate\n",
      "400        trees   shirt  z-to-xy   382    52              382        mere_correlate\n",
      "411         roof   shirt  z-to-xy   501    52              501        mere_correlate\n",
      "452         hair   shirt  z-to-xy   120    52              120        mere_correlate\n",
      "523          ear   shirt  z-to-xy   275    52              275        mere_correlate\n",
      "571      glasses   shirt  z-to-xy   163    52              163        mere_correlate\n",
      "572         face   shirt  z-to-xy   328    52              328        mere_correlate\n",
      "573       flower   shirt  z-to-xy   663    52              663        mere_correlate\n",
      "582        crowd   shirt   x-to-y   461    52              461                 cause\n",
      "591        beard   shirt  z-to-xy   732    52              732        mere_correlate\n",
      "598          hat   shirt  z-to-xy   235    52              235        mere_correlate\n",
      "677       player   shirt   x-to-y    92    52               92                 cause\n",
      "735        shirt     man   y-to-x    52    51               51                effect\n",
      "745          men   shirt   x-to-y   683    52              683                 cause\n",
      "746         nose   shirt  z-to-xy   392    52              392        mere_correlate\n",
      "766        socks   shirt  z-to-xy  1160    52             1160        mere_correlate\n",
      "771        dress   shirt  z-to-xy   482    52              482        mere_correlate\n",
      "816       people   shirt   x-to-y   541    52              541                 cause\n",
      "817        shirt  helmet  z-to-xy    52    37               37        mere_correlate\n",
      "883         head   shirt  z-to-xy   192    52              192        mere_correlate\n",
      "960        mouth   shirt  z-to-xy   453    52              453        mere_correlate\n",
      "1016       pants   shirt  z-to-xy    99    52               99        mere_correlate\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "`cause_candidate` is the id of the variable other than the effect_object_id, for which we have some information. It is 'candidate' because it could still be cause, effect or mere correlate.\n",
    "`cause_candidate_label` then tells you what the `cause_candidate` actually is.\n",
    "This is used in `test_confounder_finding.py`."
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}