Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ JobMiner is a powerful Python-based web scraping toolkit for extracting and orga

## ✨ Features

- **Enhanced CLI with TUI**: Interactive Text User Interface for user-friendly job scraping
- **Modular Architecture**: Easy-to-extend scraper system with base classes
- **Multiple Output Formats**: Export to JSON, CSV, or both
- **Database Integration**: Optional SQLite/PostgreSQL storage with search capabilities
- **CLI Interface**: Command-line tool for easy scraping operations
- **Dual Interface**: Both traditional CLI and modern TUI for different use cases
- **Real-time Progress**: Visual progress bars and status updates during scraping
- **Configuration Management**: Flexible configuration system with environment variables
- **Rate Limiting**: Built-in delays and respectful scraping practices
- **Error Handling**: Comprehensive logging and error recovery
Expand All @@ -31,6 +33,17 @@ pip install -e .

### Basic Usage

#### Enhanced TUI Interface (Recommended for Interactive Use)
```bash
# Launch interactive Text User Interface
python jobminer_cli.py tui

# TUI with options
python jobminer_cli.py tui --no-mouse # Disable mouse support
python jobminer_cli.py tui --debug # Enable debug mode
```

#### Traditional CLI (Great for Scripting)
```bash
# List available scrapers
python jobminer_cli.py list-scrapers
Expand Down
4 changes: 2 additions & 2 deletions config.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ def save_config():
"delay": 2.0,
"timeout": 30,
"max_retries": 3,
"user_agent": null,
"user_agent": None,
"headers": {}
},
"database": {
Expand All @@ -183,5 +183,5 @@ def save_config():
"echo": False
},
"log_level": "INFO",
"log_file": null
"log_file": None
}
65 changes: 61 additions & 4 deletions jobminer_cli.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
"""
JobMiner CLI - Command line interface for running job scrapers.
Enhanced with Text User Interface (TUI) for improved user experience.
"""

import click
Expand Down Expand Up @@ -62,11 +63,25 @@ def load_scraper_class(scraper_path: str):
raise ValueError(f"No scraper class found in {scraper_path}")


@click.group()
@click.group(invoke_without_command=True)
@click.version_option(version="1.0.0")
def cli():
"""JobMiner - A Python-based web scraping toolkit for job listings."""
pass
@click.pass_context
def cli(ctx):
"""
JobMiner - A Python-based web scraping toolkit for job listings.

Use 'jobminer tui' for an enhanced interactive interface,
or use the individual commands below for scripting and automation.
"""
if ctx.invoked_subcommand is None:
click.echo("πŸ” JobMiner - Interactive Job Scraping Toolkit")
click.echo()
click.echo("Quick start options:")
click.echo(" jobminer_cli.py tui # Launch interactive TUI interface")
click.echo(" jobminer_cli.py list-scrapers # List available job site scrapers")
click.echo(" jobminer_cli.py scrape [scraper] [term] # Run a specific scraper")
click.echo()
click.echo("For full help: jobminer_cli.py --help")


@cli.command()
Expand All @@ -83,6 +98,48 @@ def list_scrapers():
click.echo(f" β€’ {name} ({path})")


@cli.command()
@click.option('--no-mouse', is_flag=True, help='Disable mouse support')
@click.option('--debug', is_flag=True, help='Enable debug mode')
def tui(no_mouse, debug):
"""Launch the Enhanced Text User Interface (TUI) for interactive job scraping."""
try:
# Import TUI components
from tui.main_app import JobMinerTUIApp

click.echo("πŸš€ Starting JobMiner Enhanced CLI...")
click.echo("πŸ“ Use Ctrl+C to exit at any time")
click.echo("❓ Press 'h' for help once the interface loads")
click.echo()

# Create and configure the TUI application
app = JobMinerTUIApp()

if debug:
app.debug = True

if no_mouse:
app.mouse_enabled = False

# Run the application
app.run()

except ImportError:
click.echo("❌ TUI dependencies not found. Please install them:")
click.echo(" pip install textual rich")
sys.exit(1)
except KeyboardInterrupt:
click.echo("\nπŸ‘‹ Thanks for using JobMiner Enhanced CLI!")
sys.exit(0)
except Exception as e:
click.echo(f"❌ Error starting TUI: {e}")
if debug:
import traceback
traceback.print_exc()
click.echo("πŸ’‘ You can still use the traditional CLI commands")
sys.exit(1)


@cli.command()
@click.argument('scraper_name')
@click.argument('search_term')
Expand Down
2 changes: 2 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@ click==8.1.7
sqlalchemy==2.0.23
lxml==4.9.3
fake-useragent==1.4.0
textual==0.45.1
rich==13.7.0
9 changes: 9 additions & 0 deletions tui/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
"""
JobMiner Text User Interface (TUI) package.

This package contains the enhanced CLI interface with rich text formatting
and interactive components built using Textual and Rich libraries.
"""

__version__ = "1.0.0"
__author__ = "JobMiner Contributors"
5 changes: 5 additions & 0 deletions tui/components/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
"""
TUI Components package.

Contains reusable UI components for the JobMiner TUI interface.
"""
Loading