I MUST CONTINUE MY INVESTIGATION.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import vk_api\n",
    "import envparse\n",
    "from typing import NamedTuple, Optional\n",
    "import pathlib\n",
    "import pendulum\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VkAuth:\n",
    "    phone: str\n",
    "    password: str\n",
    "    session: Optional[vk_api.VkApi]\n",
    "\n",
    "    def __init__(self, phone, password, session):\n",
    "        self.phone: str = phone\n",
    "        self.password: str = password\n",
    "        self.session: Optional[vk_api.VkApi] = session\n",
    "\n",
    "\n",
    "    @classmethod\n",
    "    def from_env(cls):\n",
    "        env = envparse.Env()\n",
    "        env.read_envfile()\n",
    "\n",
    "        return cls(\n",
    "            phone=env.str('VK_PHONE'),\n",
    "            password=env.str('VK_PASSWORD'),\n",
    "            session=None\n",
    "        )\n",
    "\n",
    "    def get_session(self):\n",
    "        if self.session is None:\n",
    "\n",
    "            saved_config = pathlib.Path('./vk_config.v2.json')\n",
    "            if saved_config.exists():\n",
    "                self.session = vk_api.VkApi(self.phone)\n",
    "            else:    \n",
    "                def auth_handler():\n",
    "                    key = input('Enter authentication code')\n",
    "                    return key, True\n",
    "    \n",
    "                self.session = vk_api.VkApi(\n",
    "                    self.phone,\n",
    "                    self.password,\n",
    "                    auth_handler=auth_handler)\n",
    "            self.session.auth()\n",
    "        return self.session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth = VkAuth.from_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<vk_api.vk_api.VkApiMethod object at 0x7f3960170ca0>\n"
     ]
    }
   ],
   "source": [
    "session = auth.get_session()\n",
    "vk = session.get_api()\n",
    "\n",
    "print(vk)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Same but with token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_auth():\n",
    "    APP_ID = 7322078\n",
    "    SERVICE_TOKEN = \"e0a0d1a1e0a0d1a1e0a0d1a142e0cf687fee0a0e0a0d1a1bee8d7f9e3f8e17d82341955\"\n",
    "\n",
    "    session = vk_api.VkApi(\n",
    "        login=auth.phone,\n",
    "        token=SERVICE_TOKEN,\n",
    "        app_id=APP_ID\n",
    "    )\n",
    "    return session\n",
    "\n",
    "# cannot figure out how to use service token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "session.auth()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_ids = [\n",
    "    19732513, # \"Правда о прививках\"\n",
    "    457918, # Я ПРОТИВ ПРИВИВАНИЯ ДЕТЕЙ\n",
    "    9093914, # Мамы и папы против прививок!!\n",
    "    7362539, # Вся правда о прививках\n",
    "    59728906, # Вся правда о прививках!!!\n",
    "    182362555, # ИММУННЫЙ ОТВЕТ - 2 июня - Всероссийская акция\n",
    "    38532412, # Прививки: «за» и «против»\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fetch members"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# group_id_to_analyze = group_ids[1]\n",
    "group_id_to_analyze = group_ids[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_fetched = 0\n",
    "\n",
    "members = vk.groups.getMembers(\n",
    "    group_id=group_id_to_analyze,\n",
    "    fields=\"sex,bdate,city,country,education\"\n",
    ")\n",
    "total_fetched += len(members['items'])\n",
    "total = members['count']\n",
    "\n",
    "member_list = {total_fetched: members['items']}\n",
    "offset = total_fetched\n",
    "\n",
    "while total_fetched < total:\n",
    "    members = vk.groups.getMembers(\n",
    "        group_id=group_id_to_analyze,\n",
    "        fields=\"sex,bdate,city,country,education\",\n",
    "        offset=offset,\n",
    "    )\n",
    "    \n",
    "    total_fetched += len(members['items'])\n",
    "    offset += len(members['items'])\n",
    "    member_list[total_fetched] = members['items']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "member_list.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = [\n",
    "    pd.DataFrame(items)\n",
    "    for items in member_list.values()\n",
    "]\n",
    "df = pd.concat(dfs)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['university'].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datestamp = pendulum.today().strftime(\"%Y%b%d\").lower()\n",
    "\n",
    "df.to_csv(f'members_{group_id_to_analyze}_{datestamp}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_id_to_analyze"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fetch friends"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total users 80923\n",
      "Total uniq users 70966\n"
     ]
    }
   ],
   "source": [
    "fnames = !ls members_*.csv\n",
    "df = pd.concat(\n",
    "    [pd.read_csv(fname, index_col=0) for fname in fnames]\n",
    ")\n",
    "print('Total users', len(df))\n",
    "df = df.drop_duplicates(subset='id')\n",
    "print('Total uniq users', len(df.id.unique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  2699,   3696,   9602,  43808,  51342,  68107,  69285,  79825,\n",
       "       103821, 114482, 116598, 124850, 133089, 137038, 138818, 140915,\n",
       "       140956, 147188, 150032, 152212, 152962, 162198, 165712, 167242,\n",
       "       170946])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.id.values[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def by_chunk(iterable, chunk_size=1000):\n",
    "    \"\"\"\n",
    "    Basically, [.........] -> [[...], [...], [...]]\n",
    "\n",
    "    for chunk in by_chunk(big_iterable):\n",
    "        for entry in chunk:\n",
    "            process(chunk)\n",
    "    \"\"\"\n",
    "    iterable = iter(iterable)\n",
    "    while True:\n",
    "        chunk = list(itertools.islice(iterable, chunk_size))\n",
    "        if not len(chunk):\n",
    "            break\n",
    "        yield chunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from vkparsing.vk_auth_helper import fetch_user_friends_for_chunk\n",
    "from vkparsing.parse_user_friends import get_user_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids = get_user_ids('members_*.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_user_friends_for_chunk(user_ids[5:20], part=2020)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/25 [00:00<?, ?it/s]\u001b[A\n",
      "  8%|▊         | 2/25 [00:00<00:05,  4.21it/s]\u001b[A\n",
      " 12%|█▏        | 3/25 [00:01<00:07,  2.80it/s]\u001b[A\n",
      " 16%|█▌        | 4/25 [00:01<00:07,  2.67it/s]\u001b[A\n",
      " 20%|██        | 5/25 [00:01<00:07,  2.58it/s]\u001b[A\n",
      " 24%|██▍       | 6/25 [00:02<00:07,  2.56it/s]\u001b[A\n",
      " 28%|██▊       | 7/25 [00:02<00:07,  2.51it/s]\u001b[A\n",
      " 32%|███▏      | 8/25 [00:03<00:07,  2.33it/s]\u001b[A\n",
      " 36%|███▌      | 9/25 [00:03<00:06,  2.37it/s]\u001b[A\n",
      " 40%|████      | 10/25 [00:04<00:06,  2.41it/s]\u001b[A\n",
      " 44%|████▍     | 11/25 [00:04<00:05,  2.46it/s]\u001b[A\n",
      " 48%|████▊     | 12/25 [00:04<00:05,  2.48it/s]\u001b[A\n",
      " 52%|█████▏    | 13/25 [00:05<00:04,  2.49it/s]\u001b[A\n",
      " 56%|█████▌    | 14/25 [00:05<00:04,  2.47it/s]\u001b[A\n",
      " 60%|██████    | 15/25 [00:06<00:04,  2.40it/s]\u001b[A\n",
      " 64%|██████▍   | 16/25 [00:06<00:03,  2.42it/s]\u001b[A\n",
      " 68%|██████▊   | 17/25 [00:06<00:03,  2.43it/s]\u001b[A\n",
      " 72%|███████▏  | 18/25 [00:07<00:02,  2.44it/s]\u001b[A\n",
      " 76%|███████▌  | 19/25 [00:07<00:02,  2.43it/s]\u001b[A\n",
      " 80%|████████  | 20/25 [00:08<00:02,  2.42it/s]\u001b[A\n",
      " 84%|████████▍ | 21/25 [00:08<00:01,  2.44it/s]\u001b[A\n",
      " 88%|████████▊ | 22/25 [00:09<00:01,  2.37it/s]\u001b[A\n",
      " 92%|█████████▏| 23/25 [00:09<00:00,  2.41it/s]\u001b[A\n",
      " 96%|█████████▌| 24/25 [00:09<00:00,  2.41it/s]\u001b[A\n",
      "100%|██████████| 25/25 [00:10<00:00,  2.45it/s]\u001b[A\n"
     ]
    }
   ],
   "source": [
    "user_id_to_groups = defaultdict(list)\n",
    "private_groups = 0\n",
    "users_ids = df.id.values[:5]\n",
    "for user_id in users_ids:\n",
    "    total_fetched = 0\n",
    "\n",
    "    try:\n",
    "        group_list = vk.groups.get(\n",
    "            user_id=user_id,\n",
    "            filter=['groups', 'publics']\n",
    "        )\n",
    "    except vk_api.ApiError:\n",
    "        private_groups += 1\n",
    "        continue\n",
    "\n",
    "    total_fetched += len(group_list['items'])\n",
    "    total = group_list['count']\n",
    "\n",
    "    groups = {total_fetched: group_list['items']}\n",
    "    offset = total_fetched\n",
    "\n",
    "    while total_fetched < total:\n",
    "        more_members = vk.groups.get(\n",
    "            user_id=user_id,\n",
    "            offset=offset,\n",
    "        )\n",
    "    \n",
    "        total_fetched += len(members['items'])\n",
    "        offset += 1000\n",
    "        member_list[total_fetched] = members['items']\n",
    "        groups.extend(friends_list)\n",
    "    user_id_to_groups[user_id] = groups\n",
    "# ----\n",
    "# print(' '.join([\n",
    "#     f'Total private grouplists {private_groups},'\n",
    "#     f'out of {df.id.count()} total users'\n",
    "# ]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "637191 dict_keys([1031]) 1031\n"
     ]
    }
   ],
   "source": [
    "for k, v in user_id_to_groups.items():\n",
    "    print(k, v.keys(), len(list(v.values())[0]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "response = vk.wall.get(count=1)  # Используем метод wall.get\n",
    "\n",
    "if response['items']:\n",
    "    print(response['items'][0])\n",
    "\n",
    "for group_id in group_ids:\n",
    "    group_info = vk.groups.getById(group_id=group_id)\n",
    "    members = vk.groups.getMembers(group_id=group_id)\n",
    "\n",
    "    datestamp = pendulum.today().strftime('%Y%b%d').lower()\n",
    "    fname = f'members_{group_id}_on_{datestamp}.json'\n",
    "    with open(fname, 'w') as fp:\n",
    "        json.dump(\n",
    "            {\n",
    "                'group_info': group_info,\n",
    "                'members': members,\n",
    "            },\n",
    "            fp\n",
    "        )\n",
    "    print(\n",
    "        f'saved {len(members)} members and group info in as',\n",
    "        fname\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auth = VkAuth.from_env()\n",
    "main(auth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "projector2020-poetry",
   "language": "python",
   "name": "projector2020-poetry"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}