Stabilized v1.3.2

datagather_modules/data_assembler.py

@@ -66,7 +66,7 @@ def report_preprocessing(self, short_domain, report_file_type):
     def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, keywords, keywords_flag, dorking_flag, used_api_flag, snapshotting_flag, username, from_date, end_date):
         casename, db_casename, db_creation_date, robots_filepath, sitemap_filepath, sitemap_links_filepath, report_file_type, report_folder, ctime, report_ctime = self.report_preprocessing(short_domain, report_file_type)
         logging.info(f'### THIS LOG PART FOR {casename} CASE, TIME: {ctime} STARTS HERE')
-        print(Fore.GREEN + "Started scanning domain" + Style.RESET_ALL)
+        print(Fore.LIGHTMAGENTA_EX + "\n[STARTED BASIC DOMAIN SCAN]" + Style.RESET_ALL)
         print(Fore.GREEN + "[1/11] Getting domain IP address" + Style.RESET_ALL)
         ip = cp.ip_gather(short_domain)
         print(Fore.GREEN + '[2/11] Gathering WHOIS information' + Style.RESET_ALL)
@@ -111,16 +111,15 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
         total_ports = len(ports)
         total_ips = len(subdomain_ip) + 1
         total_vulns = len(vulns)
-
-        print(Fore.LIGHTMAGENTA_EX + "\n[BASIC SCAN END]\n" + Style.RESET_ALL)
+        print(Fore.LIGHTMAGENTA_EX + "[ENDED BASIC DOMAIN SCAN]\n" + Style.RESET_ALL)
         if report_file_type == 'xlsx':
             if pagesearch_flag.lower() == 'y':
                 if subdomains[0] != 'No subdomains were found':
                     to_search_array = [subdomains, social_medias, sd_socials]
-                    print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH]\n" + Style.RESET_ALL)
+                    print(Fore.LIGHTMAGENTA_EX + "[STARTED EXTENDED DOMAIN SCAN WITH PAGESEARCH]\n" + Style.RESET_ALL)
                     ps_emails_return, accessible_subdomains, emails_amount, files_counter, cookies_counter, api_keys_counter, website_elements_counter, exposed_passwords_counter, keywords_messages_list = subdomains_parser(to_search_array[0], report_folder, keywords, keywords_flag)
                     total_links_counter = accessed_links_counter = "No results because PageSearch does not gather these categories"
-                    print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH]\n" + Style.RESET_ALL)
+                    print(Fore.LIGHTMAGENTA_EX + "[ENDED EXTENDED DOMAIN SCAN WITH PAGESEARCH]\n" + Style.RESET_ALL)
                 else:
                     print(Fore.RED + "Cant start PageSearch because no subdomains were detected")
                     accessible_subdomains = files_counter = cookies_counter = api_keys_counter = website_elements_counter = exposed_passwords_counter = total_links_counter = accessed_links_counter = emails_amount = 'No results because no subdomains were found'
@@ -136,13 +135,13 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                 dorking_results = ['Google Dorking mode was not selected for this scan']
             else:
                 dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]\n" + Style.RESET_ALL)
                 dorking_status, dorking_results = dp.transfer_results_to_xlsx(table, get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]\n" + Style.RESET_ALL)
 
             api_scan_db = []
             if used_api_flag != ['Empty']:
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]\n" + Style.RESET_ALL)
                 if '1' in used_api_flag:
                     virustotal_output = api_virustotal_check(short_domain)
                     api_scan_db.append('VirusTotal')
@@ -163,7 +162,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                     securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
                 if '3' not in used_api_flag:
                     hudsonrock_output = 'No results because user did not selected HudsonRock API scan'
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]\n" + Style.RESET_ALL)
             else:
                 virustotal_output = 'No results because user did not selected VirusTotal API scan'
                 securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
@@ -173,14 +172,14 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
             if snapshotting_flag.lower() in ['s', 'p', 'w']:
                 config_values = read_config()
                 installed_browser = config_values['installed_browser']
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[STARTED DOMAIN SNAPSHOTTING]\n" + Style.RESET_ALL)
                 if snapshotting_flag.lower() == 's':
                     take_screenshot(installed_browser, url, report_folder + '//screensnapshot.png')
                 elif snapshotting_flag.lower() == 'p':
                     save_page_as_html(url, report_folder + '//domain_html_copy.html')
                 elif snapshotting_flag.lower() == 'w':
                     download_snapshot(short_domain, from_date, end_date, report_folder)
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[ENDED DOMAIN SNAPSHOTTING]\n" + Style.RESET_ALL)
             else:
                 pass
 
@@ -199,7 +198,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
             if pagesearch_flag.lower() == 'y':
                 if subdomains[0] != 'No subdomains were found':
                     to_search_array = [subdomains, social_medias, sd_socials]
-                    print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN START: PAGESEARCH]\n" + Style.RESET_ALL)
+                    print(Fore.LIGHTMAGENTA_EX + "[STARTED EXTENDED DOMAIN SCAN WITH PAGESEARCH]" + Style.RESET_ALL)
                     (
                         ps_emails_return,
                         accessible_subdomains,
@@ -214,7 +213,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                     total_links_counter = accessed_links_counter = "No results because PageSearch does not gather these categories"
                     if len(keywords_messages_list) == 0:
                         keywords_messages_list = ['No keywords were found']
-                    print(Fore.LIGHTMAGENTA_EX + "\n[EXTENDED SCAN END: PAGESEARCH]\n" + Style.RESET_ALL)
+                    print(Fore.LIGHTMAGENTA_EX + "[ENDED EXTENDED DOMAIN SCAN WITH PAGESEARCH]\n" + Style.RESET_ALL)
                 else:
                     print(Fore.RED + "Cant start PageSearch because no subdomains were detected")
                     ps_emails_return = ""
@@ -233,13 +232,13 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                 dorking_file_path = 'Google Dorking mode was not selected for this scan'
             else:
                 dorking_db_path, table = establishing_dork_db_connection(dorking_flag.lower())
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]" + Style.RESET_ALL)
                 dorking_status, dorking_file_path = dp.save_results_to_txt(report_folder, table, get_dorking_query(short_domain, dorking_db_path, table))
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: {dorking_flag.upper()} DORKING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH {dorking_flag.upper()} DORKING TABLE]\n" + Style.RESET_ALL)
 
             api_scan_db = []
             if used_api_flag != ['Empty']:
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: API SCANNING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[STARTED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]" + Style.RESET_ALL)
                 if '1' in used_api_flag:
                     virustotal_output = api_virustotal_check(short_domain)
                     api_scan_db.append('VirusTotal')
@@ -260,7 +259,7 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
                     securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
                 if '3' not in used_api_flag:
                     hudsonrock_output = 'No results because user did not selected HudsonRock API scan'
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: API SCANNING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[ENDED EXTENDED DOMAIN SCAN WITH 3RD PARTY API]\n" + Style.RESET_ALL)
             else:
                 virustotal_output = 'No results because user did not selected VirusTotal API scan'
                 securitytrails_output = 'No results because user did not selected SecurityTrails API scan'
@@ -270,14 +269,14 @@ def data_gathering(self, short_domain, url, report_file_type, pagesearch_flag, k
             if snapshotting_flag.lower() in ['s', 'p', 'w']:
                 config_values = read_config()
                 installed_browser = config_values['installed_browser']
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN START: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[STARTED DOMAIN SNAPSHOTTING]" + Style.RESET_ALL)
                 if snapshotting_flag.lower() == 's':
                     take_screenshot(installed_browser, url, report_folder + '//screensnapshot.png')
                 elif snapshotting_flag.lower() == 'p':
                     save_page_as_html(url, report_folder + '//domain_html_copy.html')
                 elif snapshotting_flag.lower() == 'w':
                     download_snapshot(short_domain, from_date, end_date, report_folder)
-                print(Fore.LIGHTMAGENTA_EX + f"\n[EXTENDED SCAN END: PAGE SNAPSHOTTING]\n" + Style.RESET_ALL)
+                print(Fore.LIGHTMAGENTA_EX + f"[ENDED DOMAIN SNAPSHOTTING]\n" + Style.RESET_ALL)
             else:
                 pass
 

dpulse.py

@@ -1,6 +1,7 @@
 import sys
 import os
 from colorama import Fore, Style, Back
+from rich.progress import Progress, SpinnerColumn, TextColumn
 
 sys.path.append('datagather_modules')
 sys.path.append('service')
@@ -75,24 +76,29 @@ def process_report(report_filetype, short_domain, url, case_comment, keywords_li
         spinner_thread.do_run = False
         spinner_thread.join()
 
-class ProgressBar(threading.Thread):
+
+class RichProgressBar(threading.Thread):
     def __init__(self):
-        super(ProgressBar, self).__init__()
+        super(RichProgressBar, self).__init__()
         self.do_run = True
 
     def run(self):
-        for char in itertools.cycle('|/-\\'):
-            if not self.do_run:
-                break
-            print(Fore.LIGHTMAGENTA_EX + Back.WHITE + char + Style.RESET_ALL, end='\r')
-            sleep(0.1)
+        with Progress(
+            SpinnerColumn(),
+            TextColumn("[magenta]Processing scan...[/magenta]"),
+            transient=True,
+        ) as progress:
+            task = progress.add_task("", total=None)
+            while self.do_run:
+                progress.update(task)
+                sleep(0.1)
 
 def run():
     while True:
         try:
             cli.print_main_menu()
             domain_patter = r'^(?!\-)(?:[a-zA-Z0-9\-]{1,63}\.)+[a-zA-Z]{2,}$'
-            choice = input(Fore.YELLOW + "Enter your choice >> ")
+            choice = input(Fore.YELLOW + "\nEnter your choice >> ")
             if choice == "1":
                 from misc import domain_precheck
                 print(Fore.GREEN + "\nImported and activated reporting modules" + Style.RESET_ALL)
@@ -208,8 +214,8 @@ def run():
                                                 end_date = str(input('Enter end date (YYYYMMDD format): '))
                                                 snapshotting_ui_mark = "Yes, domain's main page snapshotting using Wayback Machine"
                                         cli_init.print_prescan_summary(short_domain, report_filetype.upper(), pagesearch_ui_mark, dorking_ui_mark, used_api_ui, case_comment, snapshotting_ui_mark)
-                                        print(Fore.LIGHTMAGENTA_EX + "[BASIC SCAN START]\n" + Style.RESET_ALL)
-                                        spinner_thread = ProgressBar()
+                                        #print(Fore.LIGHTMAGENTA_EX + "[BASIC SCAN START]\n" + Style.RESET_ALL)
+                                        spinner_thread = RichProgressBar()
                                         spinner_thread.start()
                                         if report_filetype.lower() in ['html', 'xlsx']:
                                             process_report(report_filetype, short_domain, url, case_comment,
@@ -221,7 +227,7 @@ def run():
             elif choice == "2":
                 import configparser
                 cli.print_settings_menu()
-                choice_settings = input(Fore.YELLOW + "Enter your choice >> ")
+                choice_settings = input(Fore.YELLOW + "\nEnter your choice >> ")
                 if choice_settings == '1':
                     print_and_return_config()
                 elif choice_settings == '2':
@@ -249,7 +255,7 @@ def run():
                     continue
             elif choice == '3':
                 cli.dorking_db_manager()
-                choice_dorking = input(Fore.YELLOW + "Enter your choice >> ")
+                choice_dorking = input(Fore.YELLOW + "\nEnter your choice >> ")
                 if choice_dorking == '1':
                     from db_creator import manage_dorks
                     cli_init.print_api_db_msg()

pagesearch/pagesearch_parsers.py

@@ -65,7 +65,7 @@ def clean_bad_pdfs(ps_docs_path):
 def subdomains_parser(subdomains_list, report_folder, keywords, keywords_flag):
     report_lines: List[str] = []
     p = make_recorder(report_lines)
-    print(Fore.GREEN + "Conducting PageSearch. Please, be patient, it may take a long time\n" + Style.RESET_ALL)
+    #print(Fore.GREEN + "Conducting PageSearch. Please, be patient, it may take a long time\n" + Style.RESET_ALL)
     ps_docs_path = os.path.join(report_folder, 'ps_documents')
     if not os.path.exists(ps_docs_path):
         os.makedirs(ps_docs_path)
@@ -195,6 +195,7 @@ def subdomains_parser(subdomains_list, report_folder, keywords, keywords_flag):
         keywords_messages_list = ['No keywords were found because of None user input']
     else:
         print(Fore.GREEN + f"[+] Total {pdf_with_keywords} keywords were found in PDF files" + Style.RESET_ALL)
+    p(Fore.LIGHTGREEN_EX + "-------------------------------------------------" + Style.RESET_ALL)
 
     data_tuple = (
         ps_emails_return,

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "dpulse"
-version = "1.3.1"
+version = "1.3.2"
 description = "Convenient,fast and user-friendly collector of domain information from Open-Sources"
 authors = ["OSINT-TECHNOLOGIES <[email protected]>"]
 readme = "README.md"

reporting_modules/html_report_creation.py

@@ -88,7 +88,6 @@ def report_assembling(short_domain, url, case_comment, data_array, report_info_a
         virustotal_output = virustotal_html_prep(virustotal_output)
         securitytrails_output = securitytrails_html_prep(securitytrails_output)
 
-
         if len(ps_emails_return) > 0:
             subdomain_mails += ps_emails_return
             subdomain_mails = list(set(subdomain_mails))