diff --git a/.gitignore b/.gitignore index 075149f..f2dd7d0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ agent-os/ .claude/ .serena/ +logs/ # Custom config/config.php diff --git a/CHANGELOG.md b/CHANGELOG.md index 63fd685..c1b521f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,41 @@ All notable changes to BigDump are documented in this file. > **Note**: BigDump was originally created by Alexey Ozerov in 2003. Version 2.x is a complete MVC refactoring by w3spi5 (2025). +## [2.24] - 2025-01-05 - Smart Table Reset & Bug Fixes + +### Added in 2.24 + +- **Smart Table Reset**: Automatic DROP of tables before import + - Pre-scans SQL file for CREATE TABLE statements (supports .sql, .gz, .bz2) + - Automatically drops existing tables before import to prevent "Table already exists" errors + - New methods in `FileAnalysisService`: `findCreateTables()`, `dropTablesForFile()` + - Security: Table name validation with regex, backtick quoting + - Graceful failure: import continues even if Smart Reset fails + +- **Smart Table Reset Tests**: New test file `tests/SmartTableResetTest.php` + - 7 tests covering table extraction, gzip support, deduplication, security validation + +### Fixed in 2.24 + +- **Session cleanup after SSE error**: Session now properly cleared after import errors + - Previously, failed imports left stale session data causing silent failures on retry + - Added `clearSessionDirect()` call after error event in `sseImport()` + +- **Navigation links**: All "Back to Home" links now use `$scriptUri` + - Fixed `href="../"` in `error.php` (line 88) + - Fixed `href="/"` in `import.php` (line 345) - was going to server root! + - Fixed `href="./"` in `layout.php` and `layout_phar.php` (header logo) + - All navigation now works correctly regardless of installation path + +### Changed in 2.24 + +- **Header styling**: Theme-aware pastel gradients for light and dark modes + - Light mode: soft yellow/orange gradient (`#fcd34d → #fdba74 → #fbbf24`) + - Dark mode: muted amber/orange gradient (`#d97706 → #ea580c → #c2410c`) + - CSS class-based theming (`.header-gradient`, `.header-title`, `.header-subtitle`) + +--- + ## [2.23] - 2025-12-31 - Single-File PHAR Distribution ### Added in 2.23 diff --git a/README.md b/README.md index 4f788a1..1b4b3a9 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ -# BigDump 2.23 - Staggered MySQL Dump Importer +# BigDump 2.24 - Staggered MySQL Dump Importer [![PHP Version](https://img.shields.io/badge/php-8.1+-yellow.svg)](https://php.net/) [![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) -[![Package Version](https://img.shields.io/badge/version-2.23-blue.svg)](https://php.net/) +[![Package Version](https://img.shields.io/badge/version-2.24-blue.svg)](https://php.net/) [![Build Assets](https://img.shields.io/badge/build-GitHub_Actions-2088FF.svg)](https://github.com/w3spi5/bigdump/actions) [![PHAR](https://img.shields.io/badge/PHAR-single--file-purple.svg)](https://github.com/w3spi5/bigdump/releases)

- BigDump Logo + BigDump Logo

BigDump is a PHP tool for importing large MySQL dumps on web servers with strict execution time limits. Originally created by Alexey Ozerov in 2003, this major version 2 is a complete refactoring using object-oriented MVC architecture. @@ -153,7 +153,7 @@ This can provide **10-50x speedup** when importing the optimized file. - `.sql.gz` - Gzip compressed SQL - `.sql.bz2` - Bzip2 compressed SQL -## PHAR Distribution (v2.23) — Easiest Install +## PHAR Distribution (v2.24) — Easiest Install Download `bigdump.phar` from [Releases](https://github.com/w3spi5/bigdump/releases) and upload to your server. That's it! @@ -350,8 +350,10 @@ bigdump/ │ │ ├── css/tailwind.css │ │ └── js/*.js │ ├── icons.svg # SVG icon sprite -│ └── img/ -│ └── logo.png +│ ├── img/ +│ │ └── logo.png +│ └── docs/ +│ └── demov2.2.png # Screenshot ├── src/ │ ├── Config/Config.php │ ├── Controllers/BigDumpController.php @@ -394,8 +396,6 @@ bigdump/ │ └── workflows/ │ ├── build-assets.yml # CI asset pipeline │ └── build-phar.yml # PHAR build & release -├── docs/ -│ └── logo.png ├── CHANGELOG.md ├── LICENSE └── README.md @@ -480,10 +480,12 @@ If uploading large files (>500MB) fails with HTTP 500 error but smaller files wo - **Original**: Alexey Ozerov (http://www.ozerov.de/bigdump) — Created in 2003 - **MVC Refactoring**: Version 2 by [w3spi5](https://github.com/w3spi5) — 2025 +> 🔗 This fork is officially linked from the [original BigDump page](https://www.ozerov.de/bigdump). + --- ## Screenshots

- BigDump Screenshot + BigDump Screenshot

diff --git a/assets/img/logo.png b/assets/img/logo.png index c852c55..ffa0929 100644 Binary files a/assets/img/logo.png and b/assets/img/logo.png differ diff --git a/config/config.example.php b/config/config.example.php index 7933c25..03bfd6c 100644 --- a/config/config.example.php +++ b/config/config.example.php @@ -56,18 +56,35 @@ * - Safe for shared hosting environments * - 64KB file buffer, 2000 INSERT batch size, 16MB max batch bytes * - COMMIT after every batch + * - 5000 lines per session (v2.25+) * * 'aggressive': * - Targets 128MB PHP memory_limit (REQUIRED: 128MB+ memory_limit) * - +20-30% throughput improvement on INSERT-heavy dumps * - 128KB file buffer, 5000 INSERT batch size, 32MB max batch bytes * - COMMIT every 3 batches + * - 10000 lines per session (v2.25+) * * WARNING: If memory_limit < 128MB, aggressive mode automatically falls * back to conservative to prevent memory exhaustion. Check logs for warnings. */ 'performance_profile' => 'conservative', + /** + * Auto-aggressive mode threshold (v2.25+). + * + * Files larger than this threshold automatically use aggressive profile, + * even if conservative is configured. This provides optimal performance + * for large imports without requiring manual configuration changes. + * + * Set to 0 to disable auto-aggressive mode. + * Default: 104857600 (100MB) + * + * Requirements: + * - PHP memory_limit >= 128MB (otherwise auto-upgrade is skipped) + */ + 'auto_profile_threshold' => 104857600, // 100MB + /** * File read buffer size (in bytes). * Controls how much data is read from the SQL file per I/O operation. @@ -141,6 +158,11 @@ /** * Number of lines to process per session (base value). + * + * v2.25+ defaults (profile-dependent): + * - Conservative: 5000 (increased from 3000) + * - Aggressive: 10000 (increased from 5000) + * * With auto-tuning enabled (default), this is dynamically adjusted * based on available RAM (NVMe-optimized profiles): * < 512 MB -> 10,000 lines @@ -150,7 +172,7 @@ * < 8 GB -> 150,000 lines * > 8 GB -> 200,000 lines */ - 'linespersession' => 3000, + 'linespersession' => 5000, /** * Force a specific batch size (bypasses auto-tuning). @@ -310,6 +332,7 @@ /** * Minimum batch size for auto-tuner. + * v2.25: Increased from 3000 to 5000 for better performance. */ 'min_batch_size' => 5000, diff --git a/dist/bigdump-config.example.php b/dist/bigdump-config.example.php new file mode 100644 index 0000000..5b142be --- /dev/null +++ b/dist/bigdump-config.example.php @@ -0,0 +1,84 @@ + 'localhost', + + /** + * Database name. + */ + 'db_name' => '', + + /** + * MySQL username. + */ + 'db_username' => '', + + /** + * MySQL password. + */ + 'db_password' => '', + + /** + * Connection charset. + * Must match the dump file charset. + */ + 'db_connection_charset' => 'utf8mb4', + + // ========================================================================= + // PERFORMANCE PROFILE + // ========================================================================= + + /** + * Performance profile selection. + * 'conservative' (default): Safe for shared hosting, 64MB memory + * 'aggressive': For dedicated servers, requires 128MB+ memory + */ + 'performance_profile' => 'conservative', + + // ========================================================================= + // UPLOAD DIRECTORY + // ========================================================================= + + /** + * Upload directory for SQL dump files. + * Path is relative to the bigdump.phar location. + * Leave empty to use default 'uploads/' directory. + */ + 'upload_dir' => './uploads/', + + // ========================================================================= + // IMPORT CONFIGURATION + // ========================================================================= + + /** + * AJAX mode (true = no page refresh during import). + */ + 'ajax' => true, + + /** + * Number of lines to process per session. + */ + 'linespersession' => 3000, + + /** + * Debug mode (shows detailed error traces). + */ + 'debug' => false, +]; \ No newline at end of file diff --git a/dist/bigdump.phar b/dist/bigdump.phar new file mode 100644 index 0000000..18ccfc7 Binary files /dev/null and b/dist/bigdump.phar differ diff --git a/src/Config/Config.php b/src/Config/Config.php index 535ca18..bc682e4 100644 --- a/src/Config/Config.php +++ b/src/Config/Config.php @@ -23,6 +23,12 @@ class Config */ private array $values = []; + /** + * Temporary runtime overrides + * @var array + */ + private array $temporaryOverrides = []; + /** * Effective performance profile after validation * @var string @@ -51,12 +57,14 @@ class Config 'insert_batch_size' => 2000, 'max_batch_bytes' => 16777216, // 16MB 'commit_frequency' => 1, + 'linespersession' => 5000, // v2.25: increased from 3000 ], 'aggressive' => [ 'file_buffer_size' => 131072, // 128KB 'insert_batch_size' => 5000, 'max_batch_bytes' => 33554432, // 32MB 'commit_frequency' => 3, + 'linespersession' => 10000, // v2.25: increased from 5000 ], ]; @@ -86,15 +94,19 @@ class Config // Import configuration 'filename' => '', 'ajax' => true, - 'linespersession' => 3000, - 'auto_tuning' => true, // Enable automatic batch size optimization - 'min_batch_size' => 3000, // Minimum batch size (safety floor) - 'max_batch_size' => 50000, // Maximum batch size (ceiling) + 'linespersession' => 5000, // v2.25: increased from 3000 + 'auto_tuning' => true, // Enable automatic batch size optimization + 'min_batch_size' => 5000, // v2.25: increased from 3000 + 'max_batch_size' => 50000, // Maximum batch size (ceiling) 'delaypersession' => 0, // Performance profile (v2.19+) 'performance_profile' => 'conservative', // 'conservative' or 'aggressive' + // Auto-aggressive mode for large files (v2.25+) + // Files larger than this threshold automatically use aggressive profile + 'auto_profile_threshold' => 104857600, // 100MB + // Profile-dependent options (defaults are conservative values) // These are overridden based on effective profile during validation 'file_buffer_size' => 65536, // 64KB conservative, 128KB aggressive @@ -221,6 +233,8 @@ private function loadConfigFile(string $file): array // Performance profile options (v2.19+) 'performance_profile', 'file_buffer_size', 'insert_batch_size', 'max_batch_bytes', 'commit_frequency', + // Auto-aggressive mode (v2.25+) + 'auto_profile_threshold', ]; // Get defined variables after including the file @@ -262,7 +276,7 @@ private function validate(): void // Check numeric values if ($this->values['linespersession'] < 1) { - $this->values['linespersession'] = 3000; + $this->values['linespersession'] = 5000; } if ($this->values['max_query_lines'] < 1) { @@ -431,12 +445,19 @@ public function getProfileInfo(): array /** * Retrieves a configuration value * + * Checks temporary overrides first, then falls back to stored values. + * * @param string $key Configuration key * @param mixed $default Default value if not found * @return mixed Configuration value */ public function get(string $key, mixed $default = null): mixed { + // Check temporary overrides first + if (array_key_exists($key, $this->temporaryOverrides)) { + return $this->temporaryOverrides[$key]; + } + return $this->values[$key] ?? $default; } @@ -453,6 +474,112 @@ public function set(string $key, mixed $value): self return $this; } + /** + * Sets a temporary configuration override + * + * Temporary overrides take precedence over regular values but are not persisted. + * Useful for runtime adjustments like auto-aggressive mode for large files. + * + * When setting 'performance_profile' temporarily, this also re-applies + * profile defaults to ensure all profile-dependent settings are updated. + * + * @param string $key Configuration key + * @param mixed $value Value + * @return self + */ + public function setTemporary(string $key, mixed $value): self + { + $this->temporaryOverrides[$key] = $value; + + // If changing performance_profile, update effective profile and re-apply defaults + if ($key === 'performance_profile') { + $this->reapplyProfileForTemporaryOverride($value); + } + + return $this; + } + + /** + * Re-applies profile settings when temporary override changes performance_profile + * + * @param string $profile The new profile value + * @return void + */ + private function reapplyProfileForTemporaryOverride(string $profile): void + { + // Validate the profile + if (!in_array($profile, ['conservative', 'aggressive'], true)) { + return; + } + + // Check memory requirements for aggressive mode + if ($profile === 'aggressive') { + $memoryLimit = $this->getPhpMemoryLimitBytes(); + if ($memoryLimit !== -1 && $memoryLimit < self::AGGRESSIVE_MIN_MEMORY) { + // Cannot use aggressive, keep conservative + error_log( + "BigDump: Auto-aggressive mode requested but memory_limit is insufficient. " . + "Keeping conservative mode." + ); + unset($this->temporaryOverrides['performance_profile']); + return; + } + } + + // Update effective profile + $this->effectiveProfile = $profile; + + // Apply profile defaults as temporary overrides for profile-dependent options + // Only override if user hasn't explicitly set them + $profileDefaults = self::$profileDefaults[$profile]; + foreach ($profileDefaults as $key => $profileValue) { + // Skip if user has explicitly set this value (different from base default) + if ($this->values[$key] !== self::$defaults[$key]) { + continue; + } + // Skip if already overridden temporarily + if (array_key_exists($key, $this->temporaryOverrides) && $key !== 'performance_profile') { + continue; + } + // Apply profile default as temporary override + $this->temporaryOverrides[$key] = $profileValue; + } + } + + /** + * Clears a temporary configuration override + * + * @param string $key Configuration key to clear + * @return self + */ + public function clearTemporary(string $key): self + { + unset($this->temporaryOverrides[$key]); + return $this; + } + + /** + * Clears all temporary configuration overrides + * + * @return self + */ + public function clearAllTemporary(): self + { + $this->temporaryOverrides = []; + return $this; + } + + /** + * Checks if a temporary override exists for a key + * + * @param string $key Configuration key + * @return bool True if temporary override exists + */ + public function hasTemporary(string $key): bool + { + return array_key_exists($key, $this->temporaryOverrides); + } + /** * Checks if a configuration key exists * @@ -461,7 +588,8 @@ public function set(string $key, mixed $value): self */ public function has(string $key): bool { - return array_key_exists($key, $this->values); + return array_key_exists($key, $this->temporaryOverrides) + || array_key_exists($key, $this->values); } /** @@ -471,7 +599,7 @@ public function has(string $key): bool */ public function all(): array { - return $this->values; + return array_merge($this->values, $this->temporaryOverrides); } /** diff --git a/src/Controllers/BigDumpController.php b/src/Controllers/BigDumpController.php index e265d9e..8a8f2e2 100644 --- a/src/Controllers/BigDumpController.php +++ b/src/Controllers/BigDumpController.php @@ -245,6 +245,27 @@ public function startImport(): void return; } + // === SMART TABLE RESET === + // Pre-scan the SQL file and DROP any tables that will be created + // This prevents "Table already exists" errors on re-import + $filepath = $fileHandler->getUploadDir() . '/' . $filename; + $analysisService = new \BigDump\Services\FileAnalysisService(); + + try { + $dropResult = $analysisService->dropTablesForFile( + $filepath, + $this->importService->getDatabase() + ); + + if (!empty($dropResult['dropped'])) { + error_log("BigDump: Smart Table Reset dropped " . count($dropResult['dropped']) . " tables: " . implode(', ', $dropResult['dropped'])); + } + } catch (\Throwable $e) { + // Don't fail the import if Smart Table Reset fails + error_log("BigDump: Smart Table Reset failed (continuing anyway): " . $e->getMessage()); + } + // === END SMART TABLE RESET === + // Clear any previous session and create new one ImportSession::clearSession(); $session = ImportSession::fromRequest( @@ -501,6 +522,10 @@ public function sseImport(): void 'stats' => $stats, 'hasCreateTable' => $hasCreateTable, ]); + + // Clear session on error to prevent stale/corrupted session state + // Without this, "Back to Home" → "Import" sees invalid session and fails silently + $this->clearSessionDirect($sessionFile); } else { // Log successful import to history $this->historyService->addEntry( diff --git a/src/Core/Application.php b/src/Core/Application.php index 7ba5d96..62cbc4e 100644 --- a/src/Core/Application.php +++ b/src/Core/Application.php @@ -23,7 +23,7 @@ class Application /** * Application version. */ - public const VERSION = '2.23'; + public const VERSION = '2.24'; /** * Configuration instance. diff --git a/src/Models/SqlParser.php b/src/Models/SqlParser.php index 45b59d3..c8ec254 100644 --- a/src/Models/SqlParser.php +++ b/src/Models/SqlParser.php @@ -20,6 +20,7 @@ * - Proper handling of \\\\ (double backslash) before quotes * - DELIMITER detection only outside strings * - Protection against infinite memory accumulation + * - Optimized: skips analyzeQuotes for comments/empty lines when not in string * * @package BigDump\Models * @author w3spi5 @@ -141,6 +142,9 @@ public function getDelimiter(): string /** * Parses a line and returns complete query if available * + * OPTIMIZED (v2.25): When NOT inside a string, checks for comments/empty + * lines BEFORE calling analyzeQuotes, saving CPU cycles on comment-heavy dumps. + * * @param string $line Line to parse * @return array{query: string|null, error: string|null, delimiter_changed: bool} Parsing result */ @@ -155,21 +159,26 @@ public function parseLine(string $line): array // Normalize line endings $line = str_replace(["\r\n", "\r"], "\n", $line); - // Detect DELIMITER commands (only if not in a string) - if (!$this->inString && $this->isDelimiterCommand($line)) { - $newDelimiter = $this->extractDelimiter($line); + // OPTIMIZATION: When NOT in a string, perform early-exit checks + // before the more expensive analyzeQuotes call + if (!$this->inString) { + // Detect DELIMITER commands (only if not in a string) + if ($this->isDelimiterCommand($line)) { + $newDelimiter = $this->extractDelimiter($line); - if ($newDelimiter !== null) { - $this->delimiter = $newDelimiter; - $result['delimiter_changed'] = true; - } + if ($newDelimiter !== null) { + $this->delimiter = $newDelimiter; + $result['delimiter_changed'] = true; + } - return $result; - } + return $result; + } - // Ignore comments and empty lines (only if not in a string) - if (!$this->inString && $this->isCommentOrEmpty($line)) { - return $result; + // OPTIMIZATION (v2.25): Skip analyzeQuotes for comments and empty lines + // when NOT inside a string. This saves significant CPU on comment-heavy dumps. + if ($this->isCommentOrEmpty($line)) { + return $result; + } } // Check memory limit diff --git a/src/Services/FileAnalysisService.php b/src/Services/FileAnalysisService.php index 04a8cfd..2def4b9 100644 --- a/src/Services/FileAnalysisService.php +++ b/src/Services/FileAnalysisService.php @@ -194,6 +194,108 @@ public function hasCreateTableFor(string $filepath, string $tableName): bool return false; } } + + /** + * Find all CREATE TABLE statements in a SQL file. + * + * Scans the file (up to maxScanBytes) and extracts table names. + * Supports .sql, .gz, and .bz2 files. + * + * @param string $filepath Full path to SQL file + * @param int $maxScanBytes Maximum bytes to scan (default 50MB) + * @return array List of unique table names found + */ + public function findCreateTables(string $filepath, int $maxScanBytes = 52428800): array + { + if (!file_exists($filepath) || !is_readable($filepath)) { + return []; + } + + $lowerPath = strtolower($filepath); + $isGzip = str_ends_with($lowerPath, '.gz'); + $isBzip2 = str_ends_with($lowerPath, '.bz2'); + + try { + $content = ''; + + if ($isGzip && function_exists('gzopen')) { + $handle = @gzopen($filepath, 'rb'); + if ($handle === false) { + return []; + } + $content = @gzread($handle, $maxScanBytes); + gzclose($handle); + } elseif ($isBzip2 && function_exists('bzopen')) { + $handle = @bzopen($filepath, 'r'); + if ($handle === false) { + return []; + } + while (!feof($handle) && strlen($content) < $maxScanBytes) { + $chunk = @bzread($handle, 8192); + if ($chunk === false) { + break; + } + $content .= $chunk; + } + bzclose($handle); + } else { + $content = @file_get_contents($filepath, false, null, 0, $maxScanBytes); + } + + if ($content === false || $content === '') { + return []; + } + + // Pattern: CREATE TABLE (IF NOT EXISTS)? [`'"]?tablename[`'"]? ( + $pattern = '/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[`"\']?([a-zA-Z_][a-zA-Z0-9_]*)[`"\']?\s*\(/i'; + + if (preg_match_all($pattern, $content, $matches)) { + return array_values(array_unique($matches[1])); + } + + return []; + } catch (\Throwable $e) { + return []; + } + } + + /** + * Drop all tables that would be created by a SQL file. + * + * Scans the file for CREATE TABLE statements, validates table names, + * and executes DROP TABLE IF EXISTS for each one. + * + * @param string $filepath Full path to SQL file + * @param \BigDump\Models\Database $database Database connection + * @return array ['dropped' => [...], 'errors' => [...]] + */ + public function dropTablesForFile(string $filepath, \BigDump\Models\Database $database): array + { + $result = ['dropped' => [], 'errors' => []]; + $tables = $this->findCreateTables($filepath); + + if (empty($tables)) { + return $result; + } + + foreach ($tables as $tableName) { + // Security: validate table name format + if (!preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $tableName)) { + $result['errors'][] = "Invalid table name skipped: {$tableName}"; + continue; + } + + try { + $sql = "DROP TABLE IF EXISTS `{$tableName}`"; + $database->execute($sql); + $result['dropped'][] = $tableName; + } catch (\Throwable $e) { + $result['errors'][] = "Failed to drop {$tableName}: " . $e->getMessage(); + } + } + + return $result; + } } /** diff --git a/src/Services/ImportService.php b/src/Services/ImportService.php index af219b8..d3311d8 100644 --- a/src/Services/ImportService.php +++ b/src/Services/ImportService.php @@ -90,6 +90,12 @@ class ImportService */ private int $batchesSinceCommit = 0; + /** + * Whether auto-aggressive mode was activated for this import. + * @var bool + */ + private bool $autoAggressiveActivated = false; + /** * Get AutoTuner metrics for UI display. * @@ -98,7 +104,9 @@ class ImportService */ public function getAutoTunerMetrics(int $currentLines = 0): array { - return $this->autoTuner->getMetrics($currentLines); + $metrics = $this->autoTuner->getMetrics($currentLines); + $metrics['auto_aggressive_activated'] = $this->autoAggressiveActivated; + return $metrics; } /** @@ -112,7 +120,7 @@ public function __construct(Config $config) $this->database = new Database($config); $this->fileHandler = new FileHandler($config); $this->sqlParser = new SqlParser($config); - $this->linesPerSession = $config->get('linespersession', 3000); + $this->linesPerSession = $config->get('linespersession', 5000); // Initialize AutoTuner $this->autoTuner = new AutoTunerService($config); @@ -138,6 +146,89 @@ public function __construct(Config $config) $this->commitFrequency = (int) $config->get('commit_frequency', 1); } + /** + * Check file size and automatically upgrade to aggressive profile if needed. + * + * This implements the auto-aggressive mode feature (v2.25+): + * - Detects if file size exceeds auto_profile_threshold (default 100MB) + * - Automatically upgrades to aggressive profile for faster imports + * - Only activates if memory_limit allows aggressive mode + * + * @param string $filename Filename in uploads directory + * @return bool True if auto-aggressive mode was activated + */ + public function checkAutoAggressiveMode(string $filename): bool + { + // Get the full file path + $filepath = $this->fileHandler->getFullPath($filename); + + if (!file_exists($filepath)) { + return false; + } + + // Get file size + $fileSize = filesize($filepath); + if ($fileSize === false) { + return false; + } + + // Get the auto-aggressive threshold (default 100MB) + $threshold = (int) $this->config->get('auto_profile_threshold', 104857600); + + // Check if file exceeds threshold and current profile is conservative + if ($fileSize > $threshold && $this->config->getEffectiveProfile() === 'conservative') { + // Attempt to upgrade to aggressive profile + $this->config->setTemporary('performance_profile', 'aggressive'); + + // Check if upgrade was successful (memory requirements met) + if ($this->config->getEffectiveProfile() === 'aggressive') { + $this->autoAggressiveActivated = true; + + // Reinitialize components with new profile settings + $this->reinitializeWithNewProfile(); + + // Log the activation + $fileSizeMB = round($fileSize / 1024 / 1024, 1); + $thresholdMB = round($threshold / 1024 / 1024, 1); + error_log( + "BigDump: Auto-aggressive mode activated for {$fileSizeMB}MB file " . + "(threshold: {$thresholdMB}MB)" + ); + + return true; + } + } + + return false; + } + + /** + * Reinitialize components after profile change. + * + * Called when auto-aggressive mode is activated to update + * INSERT batcher, COMMIT frequency, and other profile-dependent settings. + * + * @return void + */ + private function reinitializeWithNewProfile(): void + { + // Update INSERT batcher with new profile settings + $insertBatchSize = (int) $this->config->get('insert_batch_size', 2000); + $maxBatchBytes = (int) $this->config->get('max_batch_bytes', 16777216); + $this->insertBatcher = new InsertBatcherService($insertBatchSize, $maxBatchBytes); + + // Update COMMIT frequency + $this->commitFrequency = (int) $this->config->get('commit_frequency', 1); + + // Update lines per session + $this->linesPerSession = (int) $this->config->get('linespersession', 5000); + + // Recalculate optimal batch size with AutoTuner + if ($this->autoTuner->isEnabled()) { + $this->linesPerSession = $this->autoTuner->calculateOptimalBatchSize(); + } + } + /** * Analyze file and initialize file-aware auto-tuning. * Call this when starting a fresh import (offset = 0). @@ -147,6 +238,9 @@ public function __construct(Config $config) */ public function analyzeFile(string $filename): ?FileAnalysisResult { + // Check for auto-aggressive mode first + $this->checkAutoAggressiveMode($filename); + if (!$this->autoTuner->isEnabled() || !$this->autoTuner->isFileAwareTuningEnabled()) { return null; } @@ -668,4 +762,14 @@ public function getInsertBatcherStatistics(): array { return $this->insertBatcher->getStatistics(); } + + /** + * Check if auto-aggressive mode was activated. + * + * @return bool True if auto-aggressive mode is active + */ + public function isAutoAggressiveActivated(): bool + { + return $this->autoAggressiveActivated; + } } diff --git a/templates/error.php b/templates/error.php index 72e3cf6..9f042ce 100644 --- a/templates/error.php +++ b/templates/error.php @@ -85,7 +85,7 @@
- + Back to Home
diff --git a/templates/home.php b/templates/home.php index c20658f..4b8fbcd 100644 --- a/templates/home.php +++ b/templates/home.php @@ -49,6 +49,12 @@
e($deleteResult['message']) ?>
+ @@ -198,7 +204,7 @@ class="btn btn-icon btn-purple" For larger files, use FTP to upload directly to e($uploadDir) ?>

-
+
diff --git a/templates/import.php b/templates/import.php index 42e32f4..fced1c9 100644 --- a/templates/import.php +++ b/templates/import.php @@ -342,7 +342,7 @@
Back to File List - Back to Home + Back to Home
hasError()): ?> @@ -387,7 +387,7 @@ class="btn btn-amber" or Start Over (resume) - Back to Home + Back to Home (DROP old tables before restarting) diff --git a/templates/layout.php b/templates/layout.php index 8b1d78d..2221d5c 100644 --- a/templates/layout.php +++ b/templates/layout.php @@ -25,22 +25,56 @@ [data-theme="light"] .icon-moon, :root:not([data-theme]) .icon-moon { display: inline; } [data-theme="dark"] .icon-moon { display: none; } [data-theme="dark"] .icon-sun { display: inline; } + + /* Header gradient - Light mode (pastel) */ + [data-theme="light"] .header-gradient, :root:not([data-theme]) .header-gradient { + background: linear-gradient(to right, #fcd34d, #fdba74, #fbbf24); + } + [data-theme="light"] .header-gradient .header-subtitle, :root:not([data-theme]) .header-gradient .header-subtitle { + color: #78350f; + } + [data-theme="light"] .header-gradient .header-title, :root:not([data-theme]) .header-gradient .header-title { + color: #451a03; + } + + /* Header gradient - Dark mode (muted pastel) */ + [data-theme="dark"] .header-gradient { + background: linear-gradient(to right, #d97706, #ea580c, #c2410c); + } + [data-theme="dark"] .header-gradient .header-subtitle { + color: #fef3c7; + } + [data-theme="dark"] .header-gradient .header-title { + color: #fffbeb; + } + + /* Dark mode toggle button */ + [data-theme="light"] .header-gradient .dark-toggle, :root:not([data-theme]) .header-gradient .dark-toggle { + background: rgba(120, 53, 15, 0.2); + border-color: rgba(120, 53, 15, 0.4); + color: #78350f; + } + [data-theme="dark"] .header-gradient .dark-toggle { + background: rgba(255, 255, 255, 0.15); + border-color: rgba(255, 255, 255, 0.3); + color: #ffffff; + }
-
+
-
- BigDump Logo + + BigDump Logo
-

BigDump ve($version) ?>

- Staggered MySQL Dump Importer +

BigDump ve($version) ?>

+ Staggered MySQL Dump Importer
-
+
- diff --git a/templates/layout_phar.php b/templates/layout_phar.php index 59d2649..0f7ed0c 100644 --- a/templates/layout_phar.php +++ b/templates/layout_phar.php @@ -24,6 +24,56 @@ [data-theme="light"] .icon-moon, :root:not([data-theme]) .icon-moon { display: inline; } [data-theme="dark"] .icon-moon { display: none; } [data-theme="dark"] .icon-sun { display: inline; } + + /* Header gradient - Light mode (pastel) */ + [data-theme="light"] .header-gradient, :root:not([data-theme]) .header-gradient { + background: linear-gradient(to right, #fcd34d, #fdba74, #fbbf24); + } + [data-theme="light"] .header-gradient .header-subtitle, :root:not([data-theme]) .header-gradient .header-subtitle { + color: #78350f; + } + [data-theme="light"] .header-gradient .header-title, :root:not([data-theme]) .header-gradient .header-title { + color: #451a03; + } + [data-theme="light"] .header-gradient .header-logo, :root:not([data-theme]) .header-gradient .header-logo { + color: #d97706; + } + [data-theme="light"] .header-gradient .header-badge, :root:not([data-theme]) .header-gradient .header-badge { + background: rgba(120, 53, 15, 0.2); + border: 1px solid rgba(120, 53, 15, 0.3); + color: #78350f; + } + + /* Header gradient - Dark mode (muted pastel) */ + [data-theme="dark"] .header-gradient { + background: linear-gradient(to right, #d97706, #ea580c, #c2410c); + } + [data-theme="dark"] .header-gradient .header-subtitle { + color: #fef3c7; + } + [data-theme="dark"] .header-gradient .header-title { + color: #fffbeb; + } + [data-theme="dark"] .header-gradient .header-logo { + color: #f59e0b; + } + [data-theme="dark"] .header-gradient .header-badge { + background: rgba(120, 53, 15, 0.4); + border: 1px solid rgba(254, 243, 199, 0.3); + color: #fef3c7; + } + + /* Dark mode toggle button */ + [data-theme="light"] .header-gradient .dark-toggle, :root:not([data-theme]) .header-gradient .dark-toggle { + background: rgba(120, 53, 15, 0.2); + border-color: rgba(120, 53, 15, 0.4); + color: #78350f; + } + [data-theme="dark"] .header-gradient .dark-toggle { + background: rgba(255, 255, 255, 0.15); + border-color: rgba(255, 255, 255, 0.3); + color: #ffffff; + } @@ -34,19 +84,19 @@
-
+
-
- -
BD
+ + +
-

BigDump ve($version) ?>

- Staggered MySQL Dump Importer - PHAR +

BigDump ve($version) ?>

+ Staggered MySQL Dump Importer + PHAR
-
+
- diff --git a/tests/AutoProfileTest.php b/tests/AutoProfileTest.php new file mode 100644 index 0000000..48efa93 --- /dev/null +++ b/tests/AutoProfileTest.php @@ -0,0 +1,472 @@ + */ + private array $failures = []; + + public function test(string $name, callable $testFn): void + { + try { + $testFn(); + $this->passed++; + echo " PASS: {$name}\n"; + } catch (Throwable $e) { + $this->failed++; + $this->failures[$name] = $e->getMessage(); + echo " FAIL: {$name}\n"; + echo " " . $e->getMessage() . "\n"; + } + } + + public function assertEquals(mixed $expected, mixed $actual, string $message = ''): void + { + if ($expected !== $actual) { + $expectedStr = var_export($expected, true); + $actualStr = var_export($actual, true); + throw new RuntimeException( + $message ?: "Expected {$expectedStr}, got {$actualStr}" + ); + } + } + + public function assertTrue(bool $condition, string $message = ''): void + { + if (!$condition) { + throw new RuntimeException($message ?: "Expected true, got false"); + } + } + + public function assertFalse(bool $condition, string $message = ''): void + { + if ($condition) { + throw new RuntimeException($message ?: "Expected false, got true"); + } + } + + public function assertGreaterThan(int|float $expected, int|float $actual, string $message = ''): void + { + if ($actual <= $expected) { + throw new RuntimeException( + $message ?: "Expected value greater than {$expected}, got {$actual}" + ); + } + } + + public function summary(): int + { + echo "\n"; + echo "==========================================\n"; + echo "Tests: " . ($this->passed + $this->failed) . ", "; + echo "Passed: {$this->passed}, "; + echo "Failed: {$this->failed}\n"; + echo "==========================================\n"; + + if ($this->failed > 0) { + echo "\nFailures:\n"; + foreach ($this->failures as $name => $message) { + echo " - {$name}: {$message}\n"; + } + } + + return $this->failed > 0 ? 1 : 0; + } +} + +/** + * Creates a temporary config file with given settings + * + * @param array $config + * @return string Path to temporary config file + */ +function createTempConfigFile(array $config): string +{ + $tempFile = sys_get_temp_dir() . '/bigdump_auto_profile_test_' . uniqid() . '.php'; + $configContent = "test('auto_profile_threshold has default value of 100MB', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + ]); + + try { + $config = new Config($configFile); + + // Default should be 100MB (104857600 bytes) + $runner->assertEquals(104857600, $config->get('auto_profile_threshold')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 2: auto_profile_threshold can be customized +// ---------------------------------------------------------------------------- +$runner->test('auto_profile_threshold can be customized', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'auto_profile_threshold' => 52428800, // 50MB + ]); + + try { + $config = new Config($configFile); + + $runner->assertEquals(52428800, $config->get('auto_profile_threshold')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 3: auto_profile_threshold can be disabled with 0 +// ---------------------------------------------------------------------------- +$runner->test('auto_profile_threshold can be disabled with 0', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'auto_profile_threshold' => 0, + ]); + + try { + $config = new Config($configFile); + + $runner->assertEquals(0, $config->get('auto_profile_threshold')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 4: setTemporary() method creates runtime override +// ---------------------------------------------------------------------------- +$runner->test('setTemporary() creates runtime config override', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'linespersession' => 5000, + ]); + + try { + $config = new Config($configFile); + + // Original value + $runner->assertEquals(5000, $config->get('linespersession')); + + // Set temporary override + $config->setTemporary('linespersession', 10000); + + // Should return overridden value + $runner->assertEquals(10000, $config->get('linespersession')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 5: setTemporary() with performance_profile cascades profile defaults +// ---------------------------------------------------------------------------- +$runner->test('setTemporary() with performance_profile cascades settings', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'performance_profile' => 'conservative', + ]); + + try { + $config = new Config($configFile); + + // Start with conservative + $runner->assertEquals('conservative', $config->getEffectiveProfile()); + $runner->assertEquals(5000, $config->get('linespersession')); // v2.25 conservative default + + // Check memory limit to determine expected behavior + $memoryLimit = ini_get('memory_limit'); + $isUnlimited = $memoryLimit === '-1' || $memoryLimit === ''; + $memLimitBytes = 0; + if (!$isUnlimited && $memoryLimit !== false) { + $unit = strtoupper(substr($memoryLimit, -1)); + $memLimitBytes = (int) $memoryLimit; + switch ($unit) { + case 'G': + $memLimitBytes *= 1024 * 1024 * 1024; + break; + case 'M': + $memLimitBytes *= 1024 * 1024; + break; + case 'K': + $memLimitBytes *= 1024; + break; + } + } + + $minRequired = 134217728; // 128MB + $canUseAggressive = $isUnlimited || $memLimitBytes >= $minRequired; + + // Set temporary override to aggressive + $config->setTemporary('performance_profile', 'aggressive'); + + if ($canUseAggressive) { + // Should upgrade to aggressive with cascaded settings + $runner->assertEquals('aggressive', $config->getEffectiveProfile()); + $runner->assertEquals(10000, $config->get('linespersession')); // v2.25 aggressive default + } else { + // Memory insufficient - should stay conservative + $runner->assertEquals('conservative', $config->getEffectiveProfile()); + } + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 6: clearTemporary() removes the override +// ---------------------------------------------------------------------------- +$runner->test('clearTemporary() removes the override', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'linespersession' => 5000, + ]); + + try { + $config = new Config($configFile); + + // Set and verify override + $config->setTemporary('linespersession', 10000); + $runner->assertEquals(10000, $config->get('linespersession')); + + // Clear override + $config->clearTemporary('linespersession'); + + // Should return original value + $runner->assertEquals(5000, $config->get('linespersession')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 7: clearAllTemporary() removes all overrides +// ---------------------------------------------------------------------------- +$runner->test('clearAllTemporary() removes all overrides', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'linespersession' => 5000, + 'min_batch_size' => 5000, + ]); + + try { + $config = new Config($configFile); + + // Set multiple overrides + $config->setTemporary('linespersession', 10000); + $config->setTemporary('min_batch_size', 8000); + + $runner->assertEquals(10000, $config->get('linespersession')); + $runner->assertEquals(8000, $config->get('min_batch_size')); + + // Clear all + $config->clearAllTemporary(); + + // Should return original values + $runner->assertEquals(5000, $config->get('linespersession')); + $runner->assertEquals(5000, $config->get('min_batch_size')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 8: hasTemporary() checks for temporary override existence +// ---------------------------------------------------------------------------- +$runner->test('hasTemporary() checks for override existence', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + ]); + + try { + $config = new Config($configFile); + + $runner->assertFalse($config->hasTemporary('linespersession')); + + $config->setTemporary('linespersession', 10000); + + $runner->assertTrue($config->hasTemporary('linespersession')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 9: User-specified values are preserved during profile cascade +// ---------------------------------------------------------------------------- +$runner->test('User-specified values preserved during profile cascade', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'performance_profile' => 'conservative', + 'linespersession' => 7500, // User override + ]); + + try { + $config = new Config($configFile); + + // User override should be preserved + $runner->assertEquals(7500, $config->get('linespersession')); + + // Check if we can use aggressive + $memoryLimit = ini_get('memory_limit'); + $isUnlimited = $memoryLimit === '-1' || $memoryLimit === ''; + $memLimitBytes = 0; + if (!$isUnlimited && $memoryLimit !== false) { + $unit = strtoupper(substr($memoryLimit, -1)); + $memLimitBytes = (int) $memoryLimit; + switch ($unit) { + case 'G': + $memLimitBytes *= 1024 * 1024 * 1024; + break; + case 'M': + $memLimitBytes *= 1024 * 1024; + break; + case 'K': + $memLimitBytes *= 1024; + break; + } + } + + $minRequired = 134217728; // 128MB + $canUseAggressive = $isUnlimited || $memLimitBytes >= $minRequired; + + // Upgrade to aggressive + $config->setTemporary('performance_profile', 'aggressive'); + + if ($canUseAggressive) { + // User-specified linespersession should still be preserved + // (not overwritten by aggressive default of 10000) + $runner->assertEquals(7500, $config->get('linespersession')); + } + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 10: New batch size defaults (v2.25) +// ---------------------------------------------------------------------------- +$runner->test('New batch size defaults - min_batch_size is 5000', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + ]); + + try { + $config = new Config($configFile); + + // v2.25: min_batch_size increased from 3000 to 5000 + $runner->assertEquals(5000, $config->get('min_batch_size')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 11: Conservative linespersession is 5000 (v2.25) +// ---------------------------------------------------------------------------- +$runner->test('Conservative linespersession is 5000 (v2.25)', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'performance_profile' => 'conservative', + ]); + + try { + $config = new Config($configFile); + + // v2.25: conservative linespersession increased from 3000 to 5000 + $runner->assertEquals(5000, $config->get('linespersession')); + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 12: Aggressive linespersession is 10000 (v2.25) +// ---------------------------------------------------------------------------- +$runner->test('Aggressive linespersession is 10000 (v2.25)', function () use ($runner) { + $configFile = createTempConfigFile([ + 'db_name' => 'test', + 'db_username' => 'test', + 'performance_profile' => 'aggressive', + ]); + + try { + $config = new Config($configFile); + + // Check if aggressive is available + if ($config->getEffectiveProfile() === 'aggressive') { + // v2.25: aggressive linespersession increased from 5000 to 10000 + $runner->assertEquals(10000, $config->get('linespersession')); + } else { + // Memory insufficient - profile was downgraded + $runner->assertTrue($config->wasProfileDowngraded()); + $runner->assertEquals(5000, $config->get('linespersession')); // Conservative fallback + } + } finally { + cleanupTempConfigFile($configFile); + } +}); + +// Output test results +exit($runner->summary()); diff --git a/tests/SmartTableResetTest.php b/tests/SmartTableResetTest.php new file mode 100644 index 0000000..b9c954f --- /dev/null +++ b/tests/SmartTableResetTest.php @@ -0,0 +1,199 @@ +test('findCreateTables extracts table names from SQL file', function () use ($runner, $testSqlFile) { + $service = new FileAnalysisService(); + $tables = $service->findCreateTables($testSqlFile); + + $runner->assertTrue(in_array('users', $tables), 'Should find users table'); + $runner->assertTrue(in_array('posts', $tables), 'Should find posts table'); + $runner->assertTrue(in_array('comments', $tables), 'Should find comments table'); + $runner->assertTrue(in_array('quoted_table', $tables), 'Should find quoted_table'); + + // Should NOT include old_table (DROP statement) + $runner->assertEquals(4, count($tables), 'Should find exactly 4 CREATE TABLE statements'); +}); + +// ============================================ +// Test: findCreateTables() with gzip file +// ============================================ +if (function_exists('gzopen')) { + $runner->test('findCreateTables works with gzip files', function () use ($runner, $testGzFile) { + $service = new FileAnalysisService(); + $tables = $service->findCreateTables($testGzFile); + + $runner->assertTrue(in_array('users', $tables), 'Should find users table in gzip'); + $runner->assertTrue(in_array('posts', $tables), 'Should find posts table in gzip'); + $runner->assertEquals(4, count($tables), 'Should find 4 tables in gzip'); + }); +} + +// ============================================ +// Test: findCreateTables() with nonexistent file +// ============================================ +$runner->test('findCreateTables returns empty array for nonexistent file', function () use ($runner) { + $service = new FileAnalysisService(); + $tables = $service->findCreateTables('/nonexistent/path/file.sql'); + + $runner->assertEquals([], $tables, 'Should return empty array'); +}); + +// ============================================ +// Test: findCreateTables() with empty file +// ============================================ +$runner->test('findCreateTables returns empty array for empty file', function () use ($runner, $fixtureDir) { + $emptyFile = $fixtureDir . '/empty_test.sql'; + file_put_contents($emptyFile, ''); + + $service = new FileAnalysisService(); + $tables = $service->findCreateTables($emptyFile); + + $runner->assertEquals([], $tables, 'Should return empty array for empty file'); + + unlink($emptyFile); +}); + +// ============================================ +// Test: findCreateTables() deduplicates tables +// ============================================ +$runner->test('findCreateTables deduplicates repeated CREATE TABLE', function () use ($runner, $fixtureDir) { + $dupFile = $fixtureDir . '/dup_test.sql'; + $dupContent = <<<'SQL' +CREATE TABLE users (id INT); +CREATE TABLE users (id INT, name VARCHAR(255)); +CREATE TABLE posts (id INT); +SQL; + file_put_contents($dupFile, $dupContent); + + $service = new FileAnalysisService(); + $tables = $service->findCreateTables($dupFile); + + $runner->assertEquals(2, count($tables), 'Should deduplicate tables'); + $runner->assertTrue(in_array('users', $tables), 'Should have users'); + $runner->assertTrue(in_array('posts', $tables), 'Should have posts'); + + unlink($dupFile); +}); + +// ============================================ +// Test: Table name validation regex +// ============================================ +$runner->test('Table names are validated with security regex', function () use ($runner) { + // Valid table names + $validNames = ['users', 'user_posts', '_private', 'Table123', 'a']; + foreach ($validNames as $name) { + $isValid = (bool) preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $name); + $runner->assertTrue($isValid, "'{$name}' should be valid"); + } + + // Invalid table names (potential SQL injection) + $invalidNames = ['users;DROP', 'table`name', "table'name", '123start', 'table name', 'table-name']; + foreach ($invalidNames as $name) { + $isValid = (bool) preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $name); + $runner->assertFalse($isValid, "'{$name}' should be invalid"); + } +}); + +// ============================================ +// Test: Integration with existing hasCreateTableFor +// ============================================ +$runner->test('findCreateTables is consistent with hasCreateTableFor for backtick tables', function () use ($runner, $testSqlFile) { + $service = new FileAnalysisService(); + + // Test tables with backticks (the format hasCreateTableFor supports) + $runner->assertTrue($service->hasCreateTableFor($testSqlFile, 'users'), 'Should find users'); + $runner->assertTrue($service->hasCreateTableFor($testSqlFile, 'posts'), 'Should find posts'); + $runner->assertTrue($service->hasCreateTableFor($testSqlFile, 'comments'), 'Should find comments'); + + // A table NOT in the file should return false + $hasIt = $service->hasCreateTableFor($testSqlFile, 'nonexistent_table'); + $runner->assertFalse($hasIt, 'hasCreateTableFor should not find nonexistent table'); +}); + +// Cleanup test fixtures +@unlink($testSqlFile); +@unlink($testGzFile); + +// Run and report +exit($runner->summary()); diff --git a/tests/SqlParserOptimizationTest.php b/tests/SqlParserOptimizationTest.php new file mode 100644 index 0000000..7e74902 --- /dev/null +++ b/tests/SqlParserOptimizationTest.php @@ -0,0 +1,442 @@ + */ + private array $failures = []; + + public function test(string $name, callable $testFn): void + { + try { + $testFn(); + $this->passed++; + echo " PASS: {$name}\n"; + } catch (Throwable $e) { + $this->failed++; + $this->failures[$name] = $e->getMessage(); + echo " FAIL: {$name}\n"; + echo " " . $e->getMessage() . "\n"; + } + } + + public function assertEquals(mixed $expected, mixed $actual, string $message = ''): void + { + if ($expected !== $actual) { + $expectedStr = var_export($expected, true); + $actualStr = var_export($actual, true); + throw new RuntimeException( + $message ?: "Expected {$expectedStr}, got {$actualStr}" + ); + } + } + + public function assertTrue(bool $condition, string $message = ''): void + { + if (!$condition) { + throw new RuntimeException($message ?: "Expected true, got false"); + } + } + + public function assertFalse(bool $condition, string $message = ''): void + { + if ($condition) { + throw new RuntimeException($message ?: "Expected false, got true"); + } + } + + public function assertNull(mixed $value, string $message = ''): void + { + if ($value !== null) { + throw new RuntimeException($message ?: "Expected null, got " . var_export($value, true)); + } + } + + public function assertNotNull(mixed $value, string $message = ''): void + { + if ($value === null) { + throw new RuntimeException($message ?: "Expected non-null value, got null"); + } + } + + public function summary(): int + { + echo "\n"; + echo "==========================================\n"; + echo "Tests: " . ($this->passed + $this->failed) . ", "; + echo "Passed: {$this->passed}, "; + echo "Failed: {$this->failed}\n"; + echo "==========================================\n"; + + if ($this->failed > 0) { + echo "\nFailures:\n"; + foreach ($this->failures as $name => $message) { + echo " - {$name}: {$message}\n"; + } + } + + return $this->failed > 0 ? 1 : 0; + } +} + +/** + * Creates a temporary config file + */ +function createTestConfig(): string +{ + $tempFile = sys_get_temp_dir() . '/bigdump_sqlparser_test_' . uniqid() . '.php'; + $config = [ + 'db_name' => 'test', + 'db_username' => 'test', + 'comment_markers' => ['#', '-- '], + ]; + file_put_contents($tempFile, "test('Comment lines with # are skipped', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $result = $parser->parseLine("# This is a comment\n"); + + $runner->assertNull($result['query']); + $runner->assertNull($result['error']); + $runner->assertFalse($result['delimiter_changed']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 2: Comment lines with -- are skipped +// ---------------------------------------------------------------------------- +$runner->test('Comment lines with -- are skipped', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $result = $parser->parseLine("-- This is a SQL comment\n"); + + $runner->assertNull($result['query']); + $runner->assertNull($result['error']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 3: Empty lines are skipped +// ---------------------------------------------------------------------------- +$runner->test('Empty lines are skipped', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $result = $parser->parseLine("\n"); + + $runner->assertNull($result['query']); + $runner->assertNull($result['error']); + + // Whitespace only + $result2 = $parser->parseLine(" \n"); + $runner->assertNull($result2['query']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 4: Multi-line string spanning comment-like content is preserved +// ---------------------------------------------------------------------------- +$runner->test('Multi-line string with comment-like content is preserved', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + // Start of INSERT with multi-line string + $result1 = $parser->parseLine("INSERT INTO test VALUES ('line1\n"); + $runner->assertNull($result1['query']); // Not complete yet + $runner->assertTrue($parser->isInString()); + + // This line looks like a comment but is inside a string + $result2 = $parser->parseLine("# This is NOT a comment - it is inside a string\n"); + $runner->assertNull($result2['query']); // Still not complete + $runner->assertTrue($parser->isInString()); // Still in string + + // Another line that looks like a comment + $result3 = $parser->parseLine("-- Also not a comment\n"); + $runner->assertNull($result3['query']); + $runner->assertTrue($parser->isInString()); + + // Close the string and complete the query + $result4 = $parser->parseLine("end of string');\n"); + $runner->assertNotNull($result4['query']); + $runner->assertFalse($parser->isInString()); + + // Verify the query contains the "comment-like" content + $query = $result4['query']; + $runner->assertTrue(strpos($query, '# This is NOT a comment') !== false); + $runner->assertTrue(strpos($query, '-- Also not a comment') !== false); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 5: Regular SQL is parsed correctly after comment +// ---------------------------------------------------------------------------- +$runner->test('Regular SQL after comments is parsed correctly', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + // Comment line + $parser->parseLine("# Initial comment\n"); + + // Empty line + $parser->parseLine("\n"); + + // SQL query + $result = $parser->parseLine("SELECT 1;\n"); + + $runner->assertNotNull($result['query']); + $runner->assertEquals('SELECT 1', $result['query']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 6: DELIMITER command is detected outside string +// ---------------------------------------------------------------------------- +$runner->test('DELIMITER command detected outside string', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $result = $parser->parseLine("DELIMITER //\n"); + + $runner->assertTrue($result['delimiter_changed']); + $runner->assertEquals('//', $parser->getDelimiter()); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 7: DELIMITER inside string is NOT treated as command +// ---------------------------------------------------------------------------- +$runner->test('DELIMITER inside string is NOT a command', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + // Start a multi-line string + $parser->parseLine("INSERT INTO test VALUES ('text\n"); + $runner->assertTrue($parser->isInString()); + + // DELIMITER inside string should NOT change delimiter + $result = $parser->parseLine("DELIMITER //\n"); + $runner->assertFalse($result['delimiter_changed']); + $runner->assertEquals(';', $parser->getDelimiter()); // Unchanged + + $runner->assertTrue($parser->isInString()); + + // Close the string + $parser->parseLine("end');\n"); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 8: Performance - many comment lines processed efficiently +// ---------------------------------------------------------------------------- +$runner->test('Many comment lines are processed efficiently', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $startTime = microtime(true); + + // Process 10000 comment lines + for ($i = 0; $i < 10000; $i++) { + $parser->parseLine("# Comment line {$i}\n"); + } + + $endTime = microtime(true); + $duration = $endTime - $startTime; + + // Should complete in under 1 second (typically < 0.1s) + $runner->assertTrue( + $duration < 1.0, + "Processing 10000 comment lines took {$duration}s (should be < 1s)" + ); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 9: Empty string content is handled correctly +// ---------------------------------------------------------------------------- +$runner->test('Empty string content is handled correctly', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $result = $parser->parseLine("INSERT INTO test VALUES ('');\n"); + + $runner->assertNotNull($result['query']); + $runner->assertEquals("INSERT INTO test VALUES ('')", $result['query']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 10: String with only whitespace +// ---------------------------------------------------------------------------- +$runner->test('String with only whitespace is preserved', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + $result = $parser->parseLine("INSERT INTO test VALUES (' ');\n"); + + $runner->assertNotNull($result['query']); + $runner->assertEquals("INSERT INTO test VALUES (' ')", $result['query']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 11: Mixed content - comments, empty lines, and SQL +// ---------------------------------------------------------------------------- +$runner->test('Mixed content with comments, empty lines, and SQL', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + // Comments and empty lines should be skipped + $parser->parseLine("# Header comment\n"); + $parser->parseLine("\n"); + $parser->parseLine("-- Another comment\n"); + $parser->parseLine("\n"); + + // This SQL should parse correctly + $result1 = $parser->parseLine("CREATE TABLE test (id INT);\n"); + $runner->assertNotNull($result1['query']); + $runner->assertEquals("CREATE TABLE test (id INT)", $result1['query']); + + // More comments + $parser->parseLine("# Middle comment\n"); + + // Another SQL + $result2 = $parser->parseLine("INSERT INTO test VALUES (1);\n"); + $runner->assertNotNull($result2['query']); + $runner->assertEquals("INSERT INTO test VALUES (1)", $result2['query']); + } finally { + cleanupTestConfig($configFile); + } +}); + +// ---------------------------------------------------------------------------- +// Test 12: Double quotes string with comment-like content +// ---------------------------------------------------------------------------- +$runner->test('Double quotes string with comment-like content', function () use ($runner) { + $configFile = createTestConfig(); + + try { + $config = new Config($configFile); + $parser = new SqlParser($config); + + // Multi-line with double quotes + $parser->parseLine('INSERT INTO test VALUES ("line1' . "\n"); + $runner->assertTrue($parser->isInString()); + + // Comment-like content inside double-quoted string + $parser->parseLine("# Not a comment\n"); + $runner->assertTrue($parser->isInString()); + + // Close and complete + $result = $parser->parseLine('end")' . ";\n"); + $runner->assertNotNull($result['query']); + $runner->assertTrue(strpos($result['query'], '# Not a comment') !== false); + } finally { + cleanupTestConfig($configFile); + } +}); + +// Output test results +exit($runner->summary());