covid-19-tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install tweepy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tweepy as tw\n",
    "import pandas as pd\n",
    "from tqdm import tqdm, notebook\n",
    "\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_colwidth', None)\n",
    "pd.set_option('display.width', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Twitter API authentication"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "consumer_api_key = os.environ[\"TWITTER_CONSUMER_API_KEY\"]\n",
    "consumer_api_secret = os.environ[\"TWITTER_CONSUMER_API_SECRET\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth = tw.OAuthHandler(consumer_api_key, consumer_api_secret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = tw.API(auth, wait_on_rate_limit=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Tweets query\n",
    "\n",
    "### 3.1. Define the query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_words = \"#covid19 -filter:retweets\"\n",
    "date_since = \"2020-03-01\"\n",
    "# Collect tweets\n",
    "tweets = tw.Cursor(api.search,\n",
    "              q=search_words,\n",
    "              lang=\"en\",\n",
    "              since=date_since).items(12452)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2. Retreive the tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12452it [16:07, 12.87it/s] \n"
     ]
    }
   ],
   "source": [
    "tweets_copy = []\n",
    "for tweet in tqdm(tweets):\n",
    "     tweets_copy.append(tweet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new tweets retrieved: 12452\n"
     ]
    }
   ],
   "source": [
    "print(f\"new tweets retrieved: {len(tweets_copy)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Populate the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 12452/12452 [3:24:19<00:00,  1.02it/s]\n"
     ]
    }
   ],
   "source": [
    "tweets_df = pd.DataFrame()\n",
    "for tweet in tqdm(tweets_copy):\n",
    "    hashtags = []\n",
    "    try:\n",
    "        for hashtag in tweet.entities[\"hashtags\"]:\n",
    "            hashtags.append(hashtag[\"text\"])\n",
    "        text = api.get_status(id=tweet.id, tweet_mode='extended').full_text\n",
    "    except:\n",
    "        pass\n",
    "    tweets_df = tweets_df.append(pd.DataFrame({'user_name': tweet.user.name, \n",
    "                                               'user_location': tweet.user.location,\\\n",
    "                                               'user_description': tweet.user.description,\n",
    "                                               'user_created': tweet.user.created_at,\n",
    "                                               'user_followers': tweet.user.followers_count,\n",
    "                                               'user_friends': tweet.user.friends_count,\n",
    "                                               'user_favourites': tweet.user.favourites_count,\n",
    "                                               'user_verified': tweet.user.verified,\n",
    "                                               'date': tweet.created_at,\n",
    "                                               'text': text, \n",
    "                                               'hashtags': [hashtags if hashtags else None],\n",
    "                                               'source': tweet.source,\n",
    "                                               'is_retweet': tweet.retweeted}, index=[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_name</th>\n",
       "      <th>user_location</th>\n",
       "      <th>user_description</th>\n",
       "      <th>user_created</th>\n",
       "      <th>user_followers</th>\n",
       "      <th>user_friends</th>\n",
       "      <th>user_favourites</th>\n",
       "      <th>user_verified</th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>source</th>\n",
       "      <th>is_retweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Jacquelyn Jhingree, Ph.D.</td>\n",
       "      <td>Canada, UK, Guyana 🌎</td>\n",
       "      <td>Chemistry PhD - @OfficialUoM,\\nScientist in #Biotech - Vaccines &amp; Drugs \\n#massspectrometry #STEM👩🏽‍🔬 \\nSoccer, Travel, Culture, Social Issues.\\nViews my own.</td>\n",
       "      <td>2013-06-24 01:57:20</td>\n",
       "      <td>723</td>\n",
       "      <td>1712</td>\n",
       "      <td>8551</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-08-30 20:31:26</td>\n",
       "      <td>Disappointing that FDA Head seems to be acceding to political intervention. \\nSuper risky to approve a #vaccine that hasn’t undergone the full course of clinical trials, with scientific rigour, to assess safety and efficacy. \\nVaccine dev. usually takes 5-10 yrs! \\n#COVID19 https://t.co/vYMdUfe1c7</td>\n",
       "      <td>[vaccine]</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",

       "      <td>Liu Xiaoming</td>\n",
       "      <td>49/51 Portland Place, London</td>\n",
       "      <td>Chinese Ambassador to the UK. \\nEmbassy account: @ChineseEmbinUK</td>\n",
       "      <td>2019-10-09 14:18:32</td>\n",
       "      <td>82103</td>\n",
       "      <td>14</td>\n",
       "      <td>48</td>\n",
       "      <td>True</td>\n",
       "      <td>2020-08-30 20:31:20</td>\n",
       "      <td>China's economy is rebounding and heralds a promising future thanks to effective #COVID19 prevention efforts. The international community has confidence in China’s economic growth and long-term development. https://t.co/BZMVkHc7Ya</td>\n",
       "      <td>[COVID19]</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Blues News Media</td>\n",
       "      <td></td>\n",
       "      <td>Blues News Media wants to save the world by calling out the global oppressors. Join and help share information!</td>\n",
       "      <td>2020-02-18 15:19:55</td>\n",
       "      <td>41</td>\n",
       "      <td>90</td>\n",
       "      <td>466</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-08-30 20:31:18</td>\n",
       "      <td>As we fast approach the fall and winter seasons the fear surrounding the #coronavirus will be ramped up again. Familiarize yourselves with what exactly is to come in this 2010 Rockefeller paper commonly know as Lock Step.\\n#COVID19\\n\\nhttps://t.co/xzhgFDWewp</td>\n",
       "      <td>[coronavirus]</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Amesh Adalja</td>\n",
       "      <td>Pittsburgh, Baltimore, NYC</td>\n",
       "      <td>Infectious disease MD working on pandemic policy, emerging infections, preventing bioterror https://t.co/Xnr2JIetFE</td>\n",
       "      <td>2010-12-16 20:41:15</td>\n",
       "      <td>31716</td>\n",
       "      <td>2008</td>\n",
       "      <td>2108</td>\n",
       "      <td>True</td>\n",
       "      <td>2020-08-30 20:31:07</td>\n",
       "      <td>In this @MSNBC interview with @LindseyReiser  I discuss various aspects of #COVID19 including vaccines and  Europe's experience https://t.co/eAgceMGtdR https://t.co/gSnlY3LPfi</td>\n",
       "      <td>[COVID19]</td>\n",
       "      <td>Twitter Web App</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>InHisGraceITrust</td>\n",
       "      <td>Central fLORiDa :)</td>\n",
       "      <td>#WhoSoEver ,YOU MUST Know #JesusIs ♫♪♥♫ #HopeLives #Meditate 📘#itsNOTaboutSIN 🕯️😇 #GRACE #TRUTH #LOVE Mysteries of HisWord ⚔️ARE YOURS! #Jn317 🙌🏽🤗🤟🏽🦋🕊</td>\n",
       "      <td>2009-06-18 20:03:30</td>\n",
       "      <td>12132</td>\n",
       "      <td>12196</td>\n",
       "      <td>21548</td>\n",
       "      <td>False</td>\n",
       "      <td>2020-08-30 20:31:06</td>\n",
       "      <td>@travis_view @J_wisecrack Yale’s Study tests different messages about vaccinating against COVID-19 once the vaccine becomes available:\\n\\nhttps://t.co/3s9tpugSys\\n\\n#FabricatedNumbers #CoVid19 #PeopleMatter #BusinessofMedicalField\\n#BusinessOfJusticeSystem\\n#JusticeForAmerica \\n#AmericaTheBrave🩸💧🇺🇸💚⚖️🔥👑🔥 https://t.co/f6vwOnLaFe</td>\n",
       "      <td>None</td>\n",
       "      <td>Twitter for iPhone</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   user_name                 user_location  \\\n",
       "0  Jacquelyn Jhingree, Ph.D.  Canada, UK, Guyana 🌎           \n",
       "0  Liu Xiaoming               49/51 Portland Place, London   \n",
       "0  Blues News Media                                          \n",
       "0  Amesh Adalja               Pittsburgh, Baltimore, NYC     \n",
       "0  InHisGraceITrust           Central fLORiDa :)             \n",
       "\n",
       "                                                                                                                                                 user_description  \\\n",
       "0  Chemistry PhD - @OfficialUoM,\\nScientist in #Biotech - Vaccines & Drugs \\n#massspectrometry #STEM👩🏽‍🔬 \\nSoccer, Travel, Culture, Social Issues.\\nViews my own.   \n",
       "0  Chinese Ambassador to the UK. \\nEmbassy account: @ChineseEmbinUK                                                                                                 \n",
       "0  Blues News Media wants to save the world by calling out the global oppressors. Join and help share information!                                                  \n",
       "0  Infectious disease MD working on pandemic policy, emerging infections, preventing bioterror https://t.co/Xnr2JIetFE                                              \n",
       "0  #WhoSoEver ,YOU MUST Know #JesusIs ♫♪♥♫ #HopeLives #Meditate 📘#itsNOTaboutSIN 🕯️😇 #GRACE #TRUTH #LOVE Mysteries of HisWord ⚔️ARE YOURS! #Jn317 🙌🏽🤗🤟🏽🦋🕊           \n",
       "\n",
       "         user_created  user_followers  user_friends  user_favourites  \\\n",
       "0 2013-06-24 01:57:20  723             1712          8551              \n",
       "0 2019-10-09 14:18:32  82103           14            48                \n",
       "0 2020-02-18 15:19:55  41              90            466               \n",
       "0 2010-12-16 20:41:15  31716           2008          2108              \n",
       "0 2009-06-18 20:03:30  12132           12196         21548             \n",
       "\n",
       "   user_verified                date  \\\n",
       "0  False         2020-08-30 20:31:26   \n",
       "0  True          2020-08-30 20:31:20   \n",
       "0  False         2020-08-30 20:31:18   \n",
       "0  True          2020-08-30 20:31:07   \n",
       "0  False         2020-08-30 20:31:06   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                        text  \\\n",
       "0  Disappointing that FDA Head seems to be acceding to political intervention. \\nSuper risky to approve a #vaccine that hasn’t undergone the full course of clinical trials, with scientific rigour, to assess safety and efficacy. \\nVaccine dev. usually takes 5-10 yrs! \\n#COVID19 https://t.co/vYMdUfe1c7                                  \n",
       "0  China's economy is rebounding and heralds a promising future thanks to effective #COVID19 prevention efforts. The international community has confidence in China’s economic growth and long-term development. https://t.co/BZMVkHc7Ya                                                                                                      \n",
       "0  As we fast approach the fall and winter seasons the fear surrounding the #coronavirus will be ramped up again. Familiarize yourselves with what exactly is to come in this 2010 Rockefeller paper commonly know as Lock Step.\\n#COVID19\\n\\nhttps://t.co/xzhgFDWewp                                                                          \n",
       "0  In this @MSNBC interview with @LindseyReiser  I discuss various aspects of #COVID19 including vaccines and  Europe's experience https://t.co/eAgceMGtdR https://t.co/gSnlY3LPfi                                                                                                                                                             \n",
       "0  @travis_view @J_wisecrack Yale’s Study tests different messages about vaccinating against COVID-19 once the vaccine becomes available:\\n\\nhttps://t.co/3s9tpugSys\\n\\n#FabricatedNumbers #CoVid19 #PeopleMatter #BusinessofMedicalField\\n#BusinessOfJusticeSystem\\n#JusticeForAmerica \\n#AmericaTheBrave🩸💧🇺🇸💚⚖️🔥👑🔥 https://t.co/f6vwOnLaFe   \n",
       "\n",
       "        hashtags              source  is_retweet  \n",
       "0  [vaccine]      Twitter for iPhone  False       \n",
       "0  [COVID19]      Twitter Web App     False       \n",
       "0  [coronavirus]  Twitter Web App     False       \n",
       "0  [COVID19]      Twitter Web App     False       \n",
       "0  None           Twitter for iPhone  False       "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Save the data\n",
    "\n",
    "### 5.1. Read past data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "past tweets: (168264, 13)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3020: DtypeWarning: Columns (7,12) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "tweets_old_df = pd.read_csv(\"covid19_tweets.csv\")\n",
    "print(f\"past tweets: {tweets_old_df.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2. Merge past and present data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new tweets: 12452 past tweets: 168264 all tweets: 180716\n"
     ]
    }
   ],
   "source": [
    "tweets_all_df = pd.concat([tweets_old_df, tweets_df], axis=0)\n",
    "print(f\"new tweets: {tweets_df.shape[0]} past tweets: {tweets_old_df.shape[0]} all tweets: {tweets_all_df.shape[0]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.3. Drop duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all tweets: (180716, 13)\n"
     ]
    }
   ],
   "source": [
    "tweets_all_df.drop_duplicates(subset = [\"user_name\", \"date\", \"text\"], inplace=True)\n",
    "print(f\"all tweets: {tweets_all_df.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.4. Export the updated data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets_all_df.to_csv(\"covid19_tweets.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}