create_data_from_whatsapp_export.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Parse whatsapp chat export between alex and yashodan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:47.363164Z",
     "start_time": "2021-08-21T12:51:47.356536Z"
    }
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_colwidth', 800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:47.863393Z",
     "start_time": "2021-08-21T12:51:47.854729Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('data_original.txt', delimiter = \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:47.940326Z",
     "start_time": "2021-08-21T12:51:47.936688Z"
    }
   },
   "outputs": [],
   "source": [
    "df.columns = ['message']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.025596Z",
     "start_time": "2021-08-21T12:51:48.022134Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_author(x):\n",
    "    if '- Alex Conway:' in x:\n",
    "        return 'alex'\n",
    "    else:\n",
    "        return 'yashodan'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.110100Z",
     "start_time": "2021-08-21T12:51:48.106148Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_content(x):\n",
    "    return ' '.join(x.split(':')[2:]).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.196834Z",
     "start_time": "2021-08-21T12:51:48.189352Z"
    }
   },
   "outputs": [],
   "source": [
    "def scrub_media(x):\n",
    "    return x.replace(\"<Media omitted>\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.275572Z",
     "start_time": "2021-08-21T12:51:48.271437Z"
    }
   },
   "outputs": [],
   "source": [
    "def scrub_urls(x):\n",
    "    return x if 'http' not in x else  ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.359844Z",
     "start_time": "2021-08-21T12:51:48.355939Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_ts(x):\n",
    "    return datetime.strptime(x.split(\" -\")[0].replace(',',''), '%Y/%m/%d %H:%M %p')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.445131Z",
     "start_time": "2021-08-21T12:51:48.439340Z"
    }
   },
   "outputs": [],
   "source": [
    "df['author'] = df['message'].apply(lambda x: get_author(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.531906Z",
     "start_time": "2021-08-21T12:51:48.522250Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content'] = df['message'].apply(lambda x: get_content(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.610303Z",
     "start_time": "2021-08-21T12:51:48.604079Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content'] = df['content'].apply(lambda x: scrub_media(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:48.869818Z",
     "start_time": "2021-08-21T12:51:48.863989Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content'] = df['content'].apply(lambda x: scrub_urls(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:49.060118Z",
     "start_time": "2021-08-21T12:51:49.055759Z"
    }
   },
   "outputs": [],
   "source": [
    "df['content_len'] = df['content'].str.len()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:49.894308Z",
     "start_time": "2021-08-21T12:51:49.889388Z"
    }
   },
   "outputs": [],
   "source": [
    "df = df[df['content_len'] > 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:50.188607Z",
     "start_time": "2021-08-21T12:51:50.172724Z"
    }
   },
   "outputs": [],
   "source": [
    "df['ts'] = df['message'].apply(lambda x: get_ts(x))\n",
    "df['ts'] = pd.to_datetime(df['ts'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:50.800324Z",
     "start_time": "2021-08-21T12:51:50.783688Z"
    }
   },
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:51.400645Z",
     "start_time": "2021-08-21T12:51:51.395126Z"
    }
   },
   "outputs": [],
   "source": [
    "df['replying_to_author'] = df['author'].shift(1)\n",
    "df['replying_to'] = df['content'].shift(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:52.623890Z",
     "start_time": "2021-08-21T12:51:52.617847Z"
    }
   },
   "outputs": [],
   "source": [
    "df = df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:51:53.820284Z",
     "start_time": "2021-08-21T12:51:53.802490Z"
    }
   },
   "outputs": [],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:54:26.424070Z",
     "start_time": "2021-08-21T12:54:26.419895Z"
    }
   },
   "outputs": [],
   "source": [
    "df['is_reply'] = df['replying_to_author'] != df['author']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:54:27.791659Z",
     "start_time": "2021-08-21T12:54:27.786850Z"
    }
   },
   "outputs": [],
   "source": [
    "df = df[df['is_reply'] == 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:54:28.757389Z",
     "start_time": "2021-08-21T12:54:28.621396Z"
    }
   },
   "outputs": [],
   "source": [
    "!tail -20 'data_original.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:56:22.053368Z",
     "start_time": "2021-08-21T12:56:22.016822Z"
    }
   },
   "outputs": [],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T13:03:18.242448Z",
     "start_time": "2021-08-21T13:03:18.231469Z"
    }
   },
   "outputs": [],
   "source": [
    "df_a = df[df['author'] == 'alex'][['content','replying_to']]\n",
    "df_a.columns = ['completion', 'prompt']\n",
    "df_a.to_csv('data_alex.csv', index=False)\n",
    "data_a = list(df_a.to_dict(orient='index').values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T13:03:18.399140Z",
     "start_time": "2021-08-21T13:03:18.386725Z"
    }
   },
   "outputs": [],
   "source": [
    "df_b = df[df['author'] == 'yashodan'][['content','replying_to']]\n",
    "df_b.columns = ['completion', 'prompt']\n",
    "df_b.to_csv('data_yashodan.csv', index=False)\n",
    "data_b = list(df_b.to_dict(orient='index').values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:59:32.932209Z",
     "start_time": "2021-08-21T12:59:32.926831Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('data_alex.jsonl', 'w') as f:\n",
    "    for item in data_a:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-08-21T12:59:38.828359Z",
     "start_time": "2021-08-21T12:59:38.788336Z"
    }
   },
   "outputs": [],
   "source": [
    "with open('data_yashodan.jsonl', 'w') as f:\n",
    "    for item in data_b:\n",
    "        f.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}