Rep_keywords_in_hf_posts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "241d6572",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:33.405913Z",
     "iopub.status.busy": "2023-10-31T13:31:33.405191Z",
     "iopub.status.idle": "2023-10-31T13:31:35.542743Z",
     "shell.execute_reply": "2023-10-31T13:31:35.541530Z"
    },
    "papermill": {
     "duration": 2.147813,
     "end_time": "2023-10-31T13:31:35.545643",
     "exception": false,
     "start_time": "2023-10-31T13:31:33.397830",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pandas as pd\n",
    "import json\n",
    "import time\n",
    "import re\n",
    "import os\n",
    "import zipfile\n",
    "from collections import defaultdict\n",
    "from nltk.tokenize import word_tokenize\n",
    "from tqdm import tqdm\n",
    "from concurrent.futures import ThreadPoolExecutor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2369ccdf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:35.558931Z",
     "iopub.status.busy": "2023-10-31T13:31:35.558049Z",
     "iopub.status.idle": "2023-10-31T13:31:35.588264Z",
     "shell.execute_reply": "2023-10-31T13:31:35.587271Z"
    },
    "papermill": {
     "duration": 0.039989,
     "end_time": "2023-10-31T13:31:35.591183",
     "exception": false,
     "start_time": "2023-10-31T13:31:35.551194",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Aliases material\n",
    "import csv\n",
    "\n",
    "def read_alias_from_csv(filename):\n",
    "    data, first_elem = [], []\n",
    "\n",
    "  # Use the 'r' mode to open the CSV file for reading\n",
    "    with open(filename, \"r\", newline=\"\") as file:\n",
    "        reader = csv.reader(file)\n",
    " \n",
    "      # Iterate through each row in the CSV file and append it to the data list\n",
    "        for row in reader:\n",
    "            data.append(row)\n",
    "\n",
    "    for line in data:\n",
    "        line[0] = line[0].upper()\n",
    "        first_elem.append(line[0])\n",
    "    return data, first_elem\n",
    "\n",
    "actor_groups_aliases, actor_first_elem_list = read_alias_from_csv('/home/anon/input/threat-aliases/threat_actors_aliases.csv')\n",
    "malware_families_aliases, malware_first_elem_list = read_alias_from_csv('/home/anon/input/threat-aliases/threat_names_aliases.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "60d7d6cf",
   "metadata": {
    "papermill": {
     "duration": 0.005177,
     "end_time": "2023-10-31T13:31:35.601927",
     "exception": false,
     "start_time": "2023-10-31T13:31:35.596750",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Generate complete list of REP keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d8a3955",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:35.614726Z",
     "iopub.status.busy": "2023-10-31T13:31:35.613887Z",
     "iopub.status.idle": "2023-10-31T13:31:37.228749Z",
     "shell.execute_reply": "2023-10-31T13:31:37.227547Z"
    },
    "papermill": {
     "duration": 1.624458,
     "end_time": "2023-10-31T13:31:37.231690",
     "exception": false,
     "start_time": "2023-10-31T13:31:35.607232",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags = [\"APT\", \"SECTEAM\", \"IDTY\", \"OS\", \"EMAIL\", \"IP\", \"DOM\", \"URL\", \"PROT\", \n",
    "         \"FILE\", \"TOOL\", \"MD5\", \"SHA1\", \"SHA2\", \"MAL\", \"ENCR\", \"VULNAME\", \"VULID\", \"ACT\"]  #\"LOC\", \"TIME\", \n",
    "\n",
    "complete_list = set()\n",
    "for t in tags:\n",
    "    file_path = f'/home/anon/input/datasets-extracted-entities/REP_extracted_entities/entity_lists/{t}_list.txt'\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            complete_list.add(line.strip())\n",
    "complete_list = sorted(complete_list, key=lambda x: (-len(x), x))\n",
    "len(complete_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24513f80",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:37.245114Z",
     "iopub.status.busy": "2023-10-31T13:31:37.244391Z",
     "iopub.status.idle": "2023-10-31T13:31:37.781115Z",
     "shell.execute_reply": "2023-10-31T13:31:37.780016Z"
    },
    "papermill": {
     "duration": 0.546897,
     "end_time": "2023-10-31T13:31:37.784181",
     "exception": false,
     "start_time": "2023-10-31T13:31:37.237284",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "chunk_size = 500\n",
    "\n",
    "def save_chunk(output_directory, count, chunk):\n",
    "    output_file = os.path.join(output_directory, f\"chunk_{count}.txt\")\n",
    "    with open(output_file, 'w', encoding='utf-8') as file:\n",
    "        file.write('\\n'.join(chunk))\n",
    "        \n",
    "output_directory = 'chunks_complete_entity_list'\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)\n",
    "\n",
    "idx_file = 1\n",
    "count = 0\n",
    "chunk = []\n",
    "for elem in tqdm(complete_list):\n",
    "    chunk.append(elem)\n",
    "    count += 1\n",
    "    if count % chunk_size == 0:\n",
    "        save_chunk(output_directory, idx_file, chunk)\n",
    "        chunk = []\n",
    "        idx_file += 1\n",
    "\n",
    "if chunk:\n",
    "    save_chunk(output_directory, idx_file, chunk)\n",
    "print('Total files:', idx_file)\n",
    "\n",
    "output_file = os.path.join(output_directory, \"complete_list.txt\")\n",
    "with open(output_file, 'w', encoding='utf-8') as file:\n",
    "    file.write('\\n'.join(complete_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c1e61b05",
   "metadata": {
    "papermill": {
     "duration": 0.005981,
     "end_time": "2023-10-31T13:31:37.796852",
     "exception": false,
     "start_time": "2023-10-31T13:31:37.790871",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Search REP keywords in HF posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1310abcc",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:37.811399Z",
     "iopub.status.busy": "2023-10-31T13:31:37.810938Z",
     "iopub.status.idle": "2023-10-31T13:31:37.835217Z",
     "shell.execute_reply": "2023-10-31T13:31:37.834285Z"
    },
    "papermill": {
     "duration": 0.034406,
     "end_time": "2023-10-31T13:31:37.837366",
     "exception": false,
     "start_time": "2023-10-31T13:31:37.802960",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def read_lines_from_file(file_path):\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            yield line.strip()\n",
    "\n",
    "def create_set_from_file(file_path):\n",
    "    return set(read_lines_from_file(file_path))\n",
    "\n",
    "def create_bulk_search_pattern(keywords_list):\n",
    "    keywords_list = sorted(keywords_list, key=lambda x: (-len(x), x))\n",
    "    #keyword_regex = r\"\\b(?:\" + \"|\".join(re.escape(word) for word in keywords_list) + r\")\\b\"\n",
    "    keyword_regex = r\"(^|\\s)[^a-zA-Z0-9]*(\" + \"|\".join(re.escape(word.lower()) for word in keywords_list) + r\")[^a-zA-Z0-9]*(\\s|$)\"\n",
    "    return re.compile(keyword_regex)\n",
    "\n",
    "def search_matches_in_chunk(chunk, bulk_search_pattern):\n",
    "    word_matching = defaultdict(list)\n",
    "    for post in chunk.itertuples(index=False):\n",
    "        title = post.threadTitle.lower()\n",
    "        content = post.tokenizedContent.lower()\n",
    "\n",
    "        matches = bulk_search_pattern.findall(content)\n",
    "        for word in matches:\n",
    "            matched_word = word[1].lower()\n",
    "            word_matching[matched_word].append({\n",
    "                'ID': post.ID,\n",
    "                'date': post.date\n",
    "            })\n",
    "    return word_matching\n",
    "\n",
    "def search_matches(input_file_path, output_file_path, df_HF):\n",
    "    # Load keywords of reports\n",
    "    keywords_list = create_set_from_file(input_file_path)\n",
    "    print('\\nNumber of report keywords: ', len(keywords_list))\n",
    "\n",
    "    print(datetime.datetime.now())\n",
    "    start = time.time()\n",
    "    print(\"Start searching for matches in file\", input_file_path)\n",
    "\n",
    "    # Convert keywords_list to lowercase for efficient matching\n",
    "    keywords_list_lower = {word.lower() for word in keywords_list}\n",
    "\n",
    "    # Compile the bulk search pattern\n",
    "    print('Create regex pattern')\n",
    "    bulk_search_pattern = create_bulk_search_pattern(keywords_list_lower)\n",
    "\n",
    "    # Divide the dataframe into smaller chunks for parallel processing\n",
    "    chunk_size = 1000\n",
    "    df_chunks = [df_HF[i:i+chunk_size] for i in range(0, df_HF.shape[0], chunk_size)]\n",
    "                \n",
    "    # Use ThreadPoolExecutor for parallel processing\n",
    "    with ThreadPoolExecutor() as executor:\n",
    "        futures = [executor.submit(search_matches_in_chunk, chunk, bulk_search_pattern) for chunk in df_chunks]\n",
    "        word_matching_dict = {}\n",
    "        for future in tqdm(futures, total=len(futures), desc='Processing Chunks'):\n",
    "            chunk_word_matching = future.result()\n",
    "            for word, matches in chunk_word_matching.items():\n",
    "                if word in word_matching_dict:\n",
    "                    word_matching_dict[word].extend(matches)\n",
    "                else:\n",
    "                    word_matching_dict[word] = matches\n",
    "\n",
    "    # Convert word_matching_dict to a list of dictionaries\n",
    "    word_matching = [{'keyword': word, 'matching posts': matches} for word, matches in word_matching_dict.items()]\n",
    "\n",
    "    end = time.time()\n",
    "    print(datetime.datetime.now())\n",
    "    print(\"Time elapsed: {} minutes\".format((end - start) / 60))\n",
    "\n",
    "    # Count non-empty lists\n",
    "    total_keywords = len(keywords_list)\n",
    "    print(f\"The total number of keywords is: {total_keywords}\")\n",
    "    non_empty_lists_count = sum(1 for entry in word_matching if entry['matching posts'])\n",
    "    percentage = (non_empty_lists_count / total_keywords) * 100\n",
    "    print(f\"The percentage of non-empty lists associated with keywords is: {percentage}%\")\n",
    "    \n",
    "    # Create a dictionary to store unique IDs for each keyword using sets\n",
    "    unique_ids = {}\n",
    "    # Iterate through the data and remove duplicates using sets\n",
    "    for item in word_matching:\n",
    "        keyword = item[\"keyword\"]\n",
    "        matching_posts = item[\"matching posts\"]\n",
    "        # Create a set to store unique IDs for the current keyword\n",
    "        unique_ids[keyword] = set()\n",
    "        # Create a list to store unique posts for the current keyword\n",
    "        unique_posts = []\n",
    "        for post in matching_posts:\n",
    "            post_id = post[\"ID\"]\n",
    "            # Check if the ID is not in the unique_ids set for this keyword\n",
    "            if post_id not in unique_ids[keyword]:\n",
    "                # Add the ID to the unique_ids set\n",
    "                unique_ids[keyword].add(post_id)\n",
    "                # Add the post to the unique_posts list\n",
    "                unique_posts.append(post)\n",
    "        # Update the matching posts for the current keyword\n",
    "        item[\"matching posts\"] = unique_posts\n",
    "\n",
    "    # Save the result\n",
    "    print(\"Saving json to %s\" % output_file_path)\n",
    "    with open(output_file_path, 'w') as file:\n",
    "        json.dump(word_matching, file, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ba03872",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:37.852137Z",
     "iopub.status.busy": "2023-10-31T13:31:37.850992Z",
     "iopub.status.idle": "2023-10-31T13:31:37.856925Z",
     "shell.execute_reply": "2023-10-31T13:31:37.855680Z"
    },
    "papermill": {
     "duration": 0.015856,
     "end_time": "2023-10-31T13:31:37.859386",
     "exception": false,
     "start_time": "2023-10-31T13:31:37.843530",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_directory_keywords = '/home/anon/working/chunks_complete_entity_list'\n",
    "input_path_HF = '/home/anon/input/datasets-extracted-entities/HF_extracted_entities/HF_extracted_entities.json'\n",
    "output_directory = '/home/anon/working/chunks_REP_keywords_in_HF_posts/'\n",
    "if not os.path.exists(output_directory):\n",
    "    os.makedirs(output_directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "692948e6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:37.873511Z",
     "iopub.status.busy": "2023-10-31T13:31:37.872731Z",
     "iopub.status.idle": "2023-10-31T13:31:56.245247Z",
     "shell.execute_reply": "2023-10-31T13:31:56.243824Z"
    },
    "papermill": {
     "duration": 18.382451,
     "end_time": "2023-10-31T13:31:56.247889",
     "exception": false,
     "start_time": "2023-10-31T13:31:37.865438",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Load HF posts\n",
    "df_HF = pd.read_json(input_path_HF)\n",
    "\n",
    "df_HF['date'] = pd.to_datetime(df_HF['date'])\n",
    "df_HF['date'] = df_HF['date'].dt.strftime('%m-%d-%Y')\n",
    "# Sort the DataFrame by the 'date' column\n",
    "df_HF.sort_values(by='date', inplace=True)\n",
    "\n",
    "# Convert title and content columns to lowercase for efficient matching\n",
    "df_HF['threadTitle'] = df_HF['threadTitle'].str.lower()\n",
    "df_HF['flatContent'] = df_HF['flatContent'].str.lower()\n",
    "print('Number of relevant posts: ', df_HF.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ddec278",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:31:56.262834Z",
     "iopub.status.busy": "2023-10-31T13:31:56.262399Z",
     "iopub.status.idle": "2023-10-31T13:32:13.338126Z",
     "shell.execute_reply": "2023-10-31T13:32:13.336459Z"
    },
    "papermill": {
     "duration": 17.086827,
     "end_time": "2023-10-31T13:32:13.341139",
     "exception": false,
     "start_time": "2023-10-31T13:31:56.254312",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "url_pattern = r'http\\S+|www\\S+|https\\S+|h\\*\\*p\\S+|h\\*\\*ps\\S+'  # Starting with http , https , www, h**p, h**ps\n",
    "delimiter_pattern = r'\\s+|[:,;?!{}\\[\\]=|]+'  # To split\n",
    "\n",
    "def process_content(row):  # Try to preserve URLs while tokenizing - Reproducing splitting used to process content to extract kw HF side\n",
    "    #content = row['threadTitle'] + ' ' + row['flatContent']\n",
    "    content = ' '.join(row['content']).lower()\n",
    "    tokens = re.split(f\"({url_pattern})\", content)  # Split the text by URLs and non-URLs\n",
    "    tokens = [token.strip() for token in tokens if token.strip()]   # Remove empty tokens and trim whitespace\n",
    "\n",
    "    # Split non-link tokens into sub_tokens\n",
    "    split_tokens = []\n",
    "    for token in tokens:\n",
    "        if not re.match(url_pattern, token):\n",
    "            sub_tokens = re.split(delimiter_pattern, token)\n",
    "            split_tokens.extend(sub_tokens)\n",
    "        else:\n",
    "            split_tokens.append(token)\n",
    "\n",
    "    return ' '.join(split_tokens)\n",
    "\n",
    "# Initialize tqdm progress bar\n",
    "pbar = tqdm(total=len(df_HF))\n",
    "\n",
    "def apply_process_content(row):\n",
    "    global pbar\n",
    "    pbar.update(1)  # Update tqdm progress bar\n",
    "    return process_content(row)\n",
    "\n",
    "df_HF['tokenizedContent'] = df_HF.apply(apply_process_content, axis=1)\n",
    "\n",
    "# Close tqdm progress bar\n",
    "pbar.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86a8d895",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T13:32:13.386456Z",
     "iopub.status.busy": "2023-10-31T13:32:13.385175Z",
     "iopub.status.idle": "2023-10-31T15:31:05.243848Z",
     "shell.execute_reply": "2023-10-31T15:31:05.242729Z"
    },
    "papermill": {
     "duration": 7131.955621,
     "end_time": "2023-10-31T15:31:05.319009",
     "exception": false,
     "start_time": "2023-10-31T13:32:13.363388",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "start = 1\n",
    "end = start + 153  # 691\n",
    "# (1, 154) - (154, 307) - (307, 460) - (460, 613) - (613, 691)\n",
    "\n",
    "end = min(end, 691)\n",
    "\n",
    "print('Searching from {0} to {1}'.format(start, end-1))\n",
    "\n",
    "for idx_file in range(start, end):  \n",
    "    iFilename = 'chunk_{0}.txt'.format(idx_file)\n",
    "    input_path = os.path.join(input_directory_keywords, iFilename)\n",
    "\n",
    "    oFilename = 'REP_keywords_in_HF_posts_{0}.json'.format(idx_file)\n",
    "    output_path = os.path.join(output_directory, oFilename)\n",
    "\n",
    "    search_matches(input_path, output_path, df_HF)\n",
    "\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f921250",
   "metadata": {
    "papermill": {
     "duration": 0.852511,
     "end_time": "2023-10-31T15:31:07.171775",
     "exception": false,
     "start_time": "2023-10-31T15:31:06.319264",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Save as ZIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "943f8c7e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-10-31T15:31:08.992025Z",
     "iopub.status.busy": "2023-10-31T15:31:08.991540Z",
     "iopub.status.idle": "2023-10-31T15:31:09.036431Z",
     "shell.execute_reply": "2023-10-31T15:31:09.034582Z"
    },
    "papermill": {
     "duration": 1.037008,
     "end_time": "2023-10-31T15:31:09.039674",
     "exception": false,
     "start_time": "2023-10-31T15:31:08.002666",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Directory path containing the output files to zip\n",
    "source_directory = output_directory\n",
    "\n",
    "# Zip file path\n",
    "zip_file_path = '/home/anon/working/REP_keywords_in_HF_posts.zip'\n",
    "\n",
    "# Create a ZIP archive\n",
    "with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "    # Loop through all files in the directory and add them to the ZIP archive\n",
    "    for root, _, files in os.walk(source_directory):\n",
    "        for file in files:\n",
    "            file_path = os.path.join(root, file)\n",
    "            zipf.write(file_path, os.path.relpath(file_path, source_directory))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 7192.01506,
   "end_time": "2023-10-31T15:31:12.638232",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-10-31T13:31:20.623172",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}