diff --git a/composer.json b/composer.json index fdfd3e1f..72c4082a 100644 --- a/composer.json +++ b/composer.json @@ -6,7 +6,9 @@ "require": { "php": ">=7.4", "ext-pdo": "*", - "ext-pdo_mysql": "*" + "ext-pdo_mysql": "*", + "wp-php-toolkit/data-liberation": "^0.4", + "wp-php-toolkit/html": "^0.4" }, "require-dev": { "phpunit/phpunit": "^10.0", diff --git a/composer.lock b/composer.lock index c410ca02..6290b56c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,8 +4,371 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "cce827c37df9ca087dbc0bec38147f5d", - "packages": [], + "content-hash": "42a96032af3d45ed4b85233b63f5bafa", + "packages": [ + { + "name": "wp-php-toolkit/bytestream", + "version": "v0.4.1", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/bytestream.git", + "reference": "664b3f6930afcd057acd3eb92370ba99853858f0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/bytestream/zipball/664b3f6930afcd057acd3eb92370ba99853858f0", + "reference": "664b3f6930afcd057acd3eb92370ba99853858f0", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "classmap": [ + "./" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "ByteStream component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/bytestream/issues", + "source": "https://github.com/wp-php-toolkit/bytestream/tree/v0.4.1" + }, + "time": "2025-09-11T23:53:08+00:00" + }, + { + "name": "wp-php-toolkit/data-liberation", + "version": "v0.4.0", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/data-liberation.git", + "reference": "0725ddef6a5fddef634e964147485827f6f72de7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/data-liberation/zipball/0725ddef6a5fddef634e964147485827f6f72de7", + "reference": "0725ddef6a5fddef634e964147485827f6f72de7", + "shasum": "" + }, + "require": { + "php": ">=7.2", + "wp-php-toolkit/bytestream": "^0.4", + "wp-php-toolkit/filesystem": "^0.4", + "wp-php-toolkit/http-client": "^0.4", + "wp-php-toolkit/xml": "^0.4" + }, + "type": "library", + "autoload": { + "files": [ + "URL/functions.php" + ], + "classmap": [ + "./", + "vendor-patched/" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "Data Liberation component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/data-liberation/issues", + "source": "https://github.com/wp-php-toolkit/data-liberation/tree/v0.4.0" + }, + "time": "2025-11-20T12:04:55+00:00" + }, + { + "name": "wp-php-toolkit/encoding", + "version": "v0.4.1", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/encoding.git", + "reference": "aafe834847cc45133836b0462f22fa1511eb3b5a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/encoding/zipball/aafe834847cc45133836b0462f22fa1511eb3b5a", + "reference": "aafe834847cc45133836b0462f22fa1511eb3b5a", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "autoload": { + "files": [ + "utf8.php", + "compat-utf8.php", + "utf8-encoder.php" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "Encoding component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/encoding/issues", + "source": "https://github.com/wp-php-toolkit/encoding/tree/v0.4.1" + }, + "time": "2025-11-20T12:04:55+00:00" + }, + { + "name": "wp-php-toolkit/filesystem", + "version": "v0.4.0", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/filesystem.git", + "reference": "ba93f3cce4400a0373118963b1c93a413e46f504" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/filesystem/zipball/ba93f3cce4400a0373118963b1c93a413e46f504", + "reference": "ba93f3cce4400a0373118963b1c93a413e46f504", + "shasum": "" + }, + "require": { + "php": ">=7.2", + "wp-php-toolkit/bytestream": "^0.4" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "files": [ + "functions.php" + ], + "psr-4": { + "WordPress\\Filesystem\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "Filesystem component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/filesystem/issues", + "source": "https://github.com/wp-php-toolkit/filesystem/tree/v0.4.0" + }, + "time": "2025-11-20T12:04:55+00:00" + }, + { + "name": "wp-php-toolkit/html", + "version": "v0.4.1", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/html.git", + "reference": "7b3863dfd82d76bd9c54e6e10a3ba317110a6e92" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/html/zipball/7b3863dfd82d76bd9c54e6e10a3ba317110a6e92", + "reference": "7b3863dfd82d76bd9c54e6e10a3ba317110a6e92", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "classmap": [ + "./" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "HTML component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/html/issues", + "source": "https://github.com/wp-php-toolkit/html/tree/v0.4.1" + }, + "time": "2025-09-08T13:34:24+00:00" + }, + { + "name": "wp-php-toolkit/http-client", + "version": "v0.4.0", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/http-client.git", + "reference": "9c04602811b91ef107d8303a66c8ebd246818665" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/http-client/zipball/9c04602811b91ef107d8303a66c8ebd246818665", + "reference": "9c04602811b91ef107d8303a66c8ebd246818665", + "shasum": "" + }, + "require": { + "php": ">=7.2", + "wp-php-toolkit/bytestream": "^0.4", + "wp-php-toolkit/data-liberation": "^0.4", + "wp-php-toolkit/filesystem": "^0.4" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "WordPress\\HttpClient\\": "" + }, + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "HttpClient component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/http-client/issues", + "source": "https://github.com/wp-php-toolkit/http-client/tree/v0.4.0" + }, + "time": "2025-11-20T12:04:55+00:00" + }, + { + "name": "wp-php-toolkit/xml", + "version": "v0.4.0", + "source": { + "type": "git", + "url": "https://github.com/wp-php-toolkit/xml.git", + "reference": "000836ad0b8eae32a61454e7127c531b3496d15f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wp-php-toolkit/xml/zipball/000836ad0b8eae32a61454e7127c531b3496d15f", + "reference": "000836ad0b8eae32a61454e7127c531b3496d15f", + "shasum": "" + }, + "require": { + "php": ">=7.2", + "wp-php-toolkit/encoding": "^0.4" + }, + "require-dev": { + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "classmap": [ + "./" + ], + "exclude-from-classmap": [ + "/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "authors": [ + { + "name": "Adam Zielinski", + "email": "adam@adamziel.com" + }, + { + "name": "WordPress Team", + "email": "wordpress@wordpress.org" + } + ], + "description": "XML component for WordPress.", + "support": { + "issues": "https://github.com/wp-php-toolkit/xml/issues", + "source": "https://github.com/wp-php-toolkit/xml/tree/v0.4.0" + }, + "time": "2025-11-20T12:04:55+00:00" + } + ], "packages-dev": [ { "name": "dealerdirect/phpcodesniffer-composer-installer", diff --git a/importer/import.php b/importer/import.php index 00fa019e..dcbc2844 100755 --- a/importer/import.php +++ b/importer/import.php @@ -15,6 +15,25 @@ require_once __DIR__ . "/../wordpress-plugin/generic/utils.php"; +// Load composer autoloader for wp-php-toolkit dependencies +$autoloader = __DIR__ . '/../vendor/autoload.php'; +if (file_exists($autoloader)) { + require_once $autoloader; +} + +// Load vendored MySQL query stream (from sqlite-database-integration PR #264) +require_once __DIR__ . '/lib/mysql-query-stream/load.php'; + +// Load WordPress function stubs (needed by wp-php-toolkit outside WordPress) +require_once __DIR__ . '/lib/wp-stubs.php'; + +// Load URL rewriting components +require_once __DIR__ . '/lib/Base64ValueScanner.php'; +require_once __DIR__ . '/lib/ContentClassifier.php'; +require_once __DIR__ . '/lib/DomainCollector.php'; +require_once __DIR__ . '/lib/SqlValueUrlRewriter.php'; +require_once __DIR__ . '/lib/SqlStatementRewriter.php'; + /** * The wire-protocol version this importer speaks. * @@ -1229,7 +1248,7 @@ public function run(array $options = []): void if (!$command) { throw new InvalidArgumentException( - "Command is required. Valid commands: files-sync, files-index, db-sync, db-index, preflight, preflight-assert", + "Command is required. Valid commands: files-sync, files-index, db-sync, db-index, db-apply, preflight, preflight-assert", ); } @@ -1239,12 +1258,13 @@ public function run(array $options = []): void "files-index", "db-sync", "db-index", + "db-apply", "preflight", "preflight-assert", ]) ) { throw new InvalidArgumentException( - "Invalid command: {$command}. Valid commands: files-sync, files-index, db-sync, db-index, preflight, preflight-assert", + "Invalid command: {$command}. Valid commands: files-sync, files-index, db-sync, db-index, db-apply, preflight, preflight-assert", ); } @@ -1359,6 +1379,30 @@ public function run(array $options = []): void return; } + // db-apply is a local-only command that doesn't need a remote server. + if ($command === "db-apply") { + if ($abort) { + $this->handle_abort($command); + return; + } + try { + $this->run_db_apply($options); + $final_status = $this->state["status"] ?? "complete"; + $this->output_progress(["status" => $final_status]); + if ($final_status === "partial") { + $this->exit_code = 2; + } + } catch (Exception $e) { + $this->output_progress([ + "status" => "error", + "error" => $e->getMessage(), + ]); + $this->write_status_file($e->getMessage()); + throw $e; + } + return; + } + // All other commands require a prior preflight run. $this->require_preflight(); @@ -1517,6 +1561,13 @@ private function handle_abort(string $command): void "FILE DELETE | {$tables_file} | abort db-sync", ); } + $domains_file = $this->state_dir . "/.import-domains.json"; + if (file_exists($domains_file)) { + unlink($domains_file); + $this->audit_log( + "FILE DELETE | {$domains_file} | abort db-sync", + ); + } break; case "db-index": @@ -1535,6 +1586,15 @@ private function handle_abort(string $command): void ); } break; + + case "db-apply": + $this->audit_log( + "RESTART | Clearing db-apply state", + true, + ); + $this->reset_state(); + $this->save_state($this->state); + break; } if ($this->is_tty && !$this->verbose_mode) { @@ -2743,6 +2803,349 @@ private function run_db_sync(): void } } + // ========================================================================= + // db-apply: Apply SQL dump to a target MySQL database with URL rewriting + // ========================================================================= + + /** + * Command: db-apply + * + * Reads db.sql, optionally rewrites URLs, and executes statements against + * a target MySQL database. Supports resumption via statement count tracking. + * + * @param array $options CLI options including target-* and url-mapping. + */ + private function run_db_apply(array $options): void + { + $sql_file = $this->state_dir . "/db.sql"; + if (!file_exists($sql_file)) { + throw new RuntimeException( + "db.sql not found in {$this->state_dir}. Run db-sync first.", + ); + } + + // Parse target database options + $target_host = $options["target_host"] ?? "127.0.0.1"; + $target_port = (int) ($options["target_port"] ?? 3306); + $target_user = $options["target_user"] ?? null; + $target_pass = $options["target_pass"] ?? ""; + $target_db = $options["target_db"] ?? null; + + if (!$target_user || !$target_db) { + throw new InvalidArgumentException( + "db-apply requires --target-user and --target-db options.", + ); + } + + // Parse URL mapping + $url_mapping = []; + if (!empty($options["url_mapping"])) { + foreach ($options["url_mapping"] as $mapping_str) { + $parts = explode("::", $mapping_str, 2); + if (count($parts) !== 2) { + throw new InvalidArgumentException( + "Invalid --url-mapping format: {$mapping_str}. Expected FROM::TO", + ); + } + $url_mapping[$parts[0]] = $parts[1]; + } + } + + // Show discovered domains if available + $domains_file = $this->state_dir . "/.import-domains.json"; + if (file_exists($domains_file)) { + $domains = json_decode(file_get_contents($domains_file), true); + if (is_array($domains) && !empty($domains)) { + $this->audit_log( + sprintf("DISCOVERED DOMAINS | %s", implode(", ", $domains)), + false, + ); + if ($this->is_tty && !$this->verbose_mode) { + echo "Discovered domains in SQL dump:\n"; + foreach ($domains as $domain) { + $mapped = isset($url_mapping[$domain]) ? " => {$url_mapping[$domain]}" : " (not mapped)"; + echo " {$domain}{$mapped}\n"; + } + echo "\n"; + } + } + } + + // Check state for resume + $state_command = $this->state["command"] ?? null; + $current_status = $state_command === "db-apply" ? ($this->state["status"] ?? null) : null; + + if ($current_status === "complete") { + throw new RuntimeException( + "db-apply already completed. Use --abort flag to re-run.", + ); + } + + $apply_state = $this->state["apply"] ?? $this->default_state()["apply"]; + $statements_executed = (int) ($apply_state["statements_executed"] ?? 0); + $bytes_read = (int) ($apply_state["bytes_read"] ?? 0); + $is_resume = $current_status === "in_progress" && $statements_executed > 0; + + if ($is_resume) { + $this->audit_log( + sprintf( + "RESUME db-apply | statements=%d | bytes_read=%d", + $statements_executed, + $bytes_read, + ), + true, + ); + if ($this->is_tty && !$this->verbose_mode) { + echo "Resuming db-apply (executed: {$statements_executed} statements)\n"; + } + } else { + $this->state["command"] = "db-apply"; + $this->state["status"] = "in_progress"; + $this->state["apply"] = $this->default_state()["apply"]; + if (!empty($url_mapping)) { + $this->state["apply"]["url_mapping"] = $url_mapping; + } + $this->save_state($this->state); + $statements_executed = 0; + $bytes_read = 0; + + $this->audit_log("START db-apply", true); + if ($this->is_tty && !$this->verbose_mode) { + echo "Starting db-apply\n"; + } + } + + // On resume, use the persisted URL mapping if none provided on CLI + if (empty($url_mapping) && !empty($apply_state["url_mapping"])) { + $url_mapping = $apply_state["url_mapping"]; + } + + // Set up SQL statement rewriter if we have URL mappings + $stmt_rewriter = null; + if (!empty($url_mapping)) { + $stmt_rewriter = new SqlStatementRewriter( + new SqlValueUrlRewriter($url_mapping), + ); + $this->audit_log( + sprintf( + "URL MAPPING | %d mapping(s): %s", + count($url_mapping), + implode(", ", array_map( + fn($from, $to) => "{$from} => {$to}", + array_keys($url_mapping), + array_values($url_mapping), + )), + ), + false, + ); + } + + // Connect to target MySQL + $dsn = "mysql:host={$target_host};port={$target_port};dbname={$target_db};charset=utf8mb4"; + try { + $pdo = new PDO($dsn, $target_user, $target_pass, [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::MYSQL_ATTR_LOCAL_INFILE => false, + ]); + } catch (PDOException $e) { + throw new RuntimeException( + "Cannot connect to target database: " . $e->getMessage(), + ); + } + + $this->audit_log( + sprintf( + "CONNECTED | host=%s port=%d db=%s user=%s", + $target_host, + $target_port, + $target_db, + $target_user, + ), + false, + ); + + // Stream db.sql through the query stream and execute + $query_stream = new \WP_MySQL_Naive_Query_Stream(); + $sql_handle = fopen($sql_file, "r"); + if (!$sql_handle) { + throw new RuntimeException("Cannot open SQL file: {$sql_file}"); + } + + $sql_file_size = filesize($sql_file); + $total_bytes_read = 0; + $stmt_count = 0; + $skipped = 0; + $save_every = 100; + $stmts_since_save = 0; + + // If resuming, seek to saved position and skip already-executed statements + $stmts_to_skip = $statements_executed; + if ($bytes_read > 0 && $bytes_read < $sql_file_size) { + fseek($sql_handle, $bytes_read); + $total_bytes_read = $bytes_read; + } elseif ($stmts_to_skip > 0) { + // Can't seek — need to scan from beginning and skip statements + $bytes_read = 0; + } + + $this->output_progress([ + "status" => "starting", + "phase" => "db-apply", + ]); + + try { + $chunk_size = 64 * 1024; // 64KB read chunks + + while (!feof($sql_handle)) { + // Check shutdown + if ($this->shutdown_requested) { + $this->audit_log("SHUTDOWN REQUESTED | saving state", true); + break; + } + if (function_exists("pcntl_signal_dispatch")) { + pcntl_signal_dispatch(); + } + + $data = fread($sql_handle, $chunk_size); + if ($data === false || $data === '') { + break; + } + $total_bytes_read += strlen($data); + $query_stream->append_sql($data); + + while ($query_stream->next_query()) { + $query = $query_stream->get_query(); + $stmt_count++; + + // Skip already-executed statements on resume + if ($stmts_to_skip > 0) { + $stmts_to_skip--; + continue; + } + + // Rewrite URLs if mapping is configured + if ($stmt_rewriter) { + $query = $stmt_rewriter->rewrite($query); + } + + // Execute against target database + try { + $pdo->exec($query); + } catch (PDOException $e) { + $this->audit_log( + sprintf( + "SQL ERROR | stmt=%d | %s | query=%.200s", + $stmt_count, + $e->getMessage(), + $query, + ), + true, + ); + throw new RuntimeException( + "SQL execution error at statement {$stmt_count}: " . + $e->getMessage(), + ); + } + + $statements_executed++; + $stmts_since_save++; + + // Save state periodically + if ($stmts_since_save >= $save_every) { + $this->state["apply"]["statements_executed"] = $statements_executed; + $this->state["apply"]["bytes_read"] = $total_bytes_read; + $this->save_state($this->state); + $stmts_since_save = 0; + + // Progress output + $pct = $sql_file_size > 0 + ? round(100 * $total_bytes_read / $sql_file_size, 1) + : 0; + $this->output_progress([ + "phase" => "db-apply", + "statements_executed" => $statements_executed, + "bytes_read" => $total_bytes_read, + "bytes_total" => $sql_file_size, + "pct" => $pct, + ]); + } + } + } + + // Drain any remaining buffered query + $query_stream->mark_input_complete(); + while ($query_stream->next_query()) { + $query = $query_stream->get_query(); + $stmt_count++; + + if ($stmts_to_skip > 0) { + $stmts_to_skip--; + continue; + } + + if ($stmt_rewriter) { + $query = $stmt_rewriter->rewrite($query); + } + + try { + $pdo->exec($query); + } catch (PDOException $e) { + $this->audit_log( + sprintf( + "SQL ERROR | stmt=%d | %s | query=%.200s", + $stmt_count, + $e->getMessage(), + $query, + ), + true, + ); + throw new RuntimeException( + "SQL execution error at statement {$stmt_count}: " . + $e->getMessage(), + ); + } + + $statements_executed++; + } + + if ($this->shutdown_requested) { + // Save partial progress + $this->state["apply"]["statements_executed"] = $statements_executed; + $this->state["apply"]["bytes_read"] = $total_bytes_read; + $this->state["status"] = "partial"; + $this->save_state($this->state); + $this->audit_log( + sprintf( + "PARTIAL db-apply | %d statements executed", + $statements_executed, + ), + true, + ); + } else { + // Mark complete + $this->state["apply"]["statements_executed"] = $statements_executed; + $this->state["apply"]["bytes_read"] = $total_bytes_read; + $this->state["status"] = "complete"; + $this->save_state($this->state); + + $this->audit_log( + sprintf( + "db-apply complete | %d statements executed", + $statements_executed, + ), + true, + ); + + if ($this->is_tty && !$this->verbose_mode) { + echo "db-apply complete ({$statements_executed} statements executed)\n"; + } + } + } finally { + fclose($sql_handle); + } + } + /** * Command: db-index * @@ -4095,6 +4498,37 @@ private function download_sql(): void ); } + // Domain discovery: scan SQL for URLs during download (file mode only) + $query_stream = ($mode === "file" && class_exists('WP_MySQL_Naive_Query_Stream')) + ? new \WP_MySQL_Naive_Query_Stream() + : null; + $domain_collector = ($mode === "file" && class_exists('DomainCollector')) + ? new \DomainCollector() + : null; + $domains_file = $this->state_dir . "/.import-domains.json"; + + // Auto-detect the source site domain from the export URL so it + // always appears in .import-domains.json even if the SQL dump + // hasn't been fully scanned yet. + if ($domain_collector) { + $parsed_url = parse_url($this->remote_url); + if ($parsed_url && isset($parsed_url['scheme'], $parsed_url['host'])) { + $source_origin = $parsed_url['scheme'] . '://' . $parsed_url['host']; + if (!empty($parsed_url['port'])) { + $source_origin .= ':' . $parsed_url['port']; + } + $domain_collector->merge([$source_origin]); + } + } + + // Load previously discovered domains (from earlier partial downloads) + if ($domain_collector && file_exists($domains_file)) { + $prev = json_decode(file_get_contents($domains_file), true); + if (is_array($prev)) { + $domain_collector->merge($prev); + } + } + // Log current progress at start of request $has_cursor = $cursor !== null; $this->audit_log( @@ -4122,7 +4556,10 @@ private function download_sql(): void $mysql_conn, &$sql_buffer, &$sql_bytes_written, - $context + $context, + $query_stream, + $domain_collector, + $domains_file ) { // Check if shutdown was requested if ($this->shutdown_requested) { @@ -4152,6 +4589,20 @@ private function download_sql(): void $this->state["sql_bytes"] = $sql_bytes_written; $this->save_state($this->state); $this->chunks_since_save = 0; + + // Also persist discovered domains so they survive crashes. + // On resume, the SQL download picks up from the cursor, + // skipping already-downloaded data — so domains from that + // earlier data would be lost without periodic saves. + if ($domain_collector) { + $domains = $domain_collector->get_domains(); + if (!empty($domains)) { + file_put_contents( + $domains_file, + json_encode($domains, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n", + ); + } + } } $chunk_type = $chunk["headers"]["x-chunk-type"] ?? ""; @@ -4170,6 +4621,15 @@ private function download_sql(): void ); } $sql_bytes_written += $bytes; + + // Feed data to query stream for domain discovery + if ($query_stream && $domain_collector) { + $query_stream->append_sql($data); + $this->drain_query_stream_for_domains( + $query_stream, + $domain_collector, + ); + } break; case "stdout": @@ -4266,6 +4726,31 @@ private function download_sql(): void $this->state["sql_bytes"] = $complete ? null : $sql_bytes_written; $this->save_state($this->state); } + + // Drain any remaining statements after download completes + if ($query_stream && $domain_collector) { + $query_stream->mark_input_complete(); + $this->drain_query_stream_for_domains( + $query_stream, + $domain_collector, + ); + + // Save discovered domains + $domains = $domain_collector->get_domains(); + if (!empty($domains)) { + file_put_contents( + $domains_file, + json_encode($domains, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES) . "\n", + ); + $this->audit_log( + sprintf( + "DOMAINS DISCOVERED | %d unique domains saved to .import-domains.json", + count($domains), + ), + false, + ); + } + } } finally { if ($sql_handle) { fclose($sql_handle); @@ -4284,6 +4769,53 @@ private function download_sql(): void } } + /** + * Drain complete SQL statements from a query stream and scan their + * base64-decoded values for URL domains. + */ + private function drain_query_stream_for_domains( + \WP_MySQL_Naive_Query_Stream $query_stream, + \DomainCollector $domain_collector + ) { + while ($query_stream->next_query()) { + $query = $query_stream->get_query(); + // Only scan INSERT statements (they contain data values). + if (!self::sql_starts_with_token($query, \WP_MySQL_Lexer::INSERT_SYMBOL)) { + continue; + } + // Only scan statements with base64 values + if (strpos($query, "FROM_BASE64(") === false) { + continue; + } + $matches = \Base64ValueScanner::scan($query); + foreach ($matches as $match) { + $domain_collector->scan($match['value']); + } + } + } + + /** + * Check whether a SQL statement's first keyword token matches a given token ID. + * Skips leading whitespace and comments, so "/* ... *​/ INSERT INTO ..." is handled. + */ + private static function sql_starts_with_token(string $sql, int $expected_token_id): bool + { + $lexer = new \WP_MySQL_Lexer($sql); + while ($lexer->next_token()) { + $token = $lexer->get_token(); + if ( + $token->id === \WP_MySQL_Lexer::WHITESPACE + || $token->id === \WP_MySQL_Lexer::COMMENT + || $token->id === \WP_MySQL_Lexer::MYSQL_COMMENT_START + || $token->id === \WP_MySQL_Lexer::MYSQL_COMMENT_END + ) { + continue; + } + return $token->id === $expected_token_id; + } + return false; + } + /** * Download table stats from the db_index endpoint. */ @@ -5931,6 +6463,12 @@ private function default_state(): array "current_file_bytes" => null, // Expected bytes written so far // Crash recovery: track SQL file size "sql_bytes" => null, // Expected SQL file size + // db-apply state + "apply" => [ + "statements_executed" => 0, + "bytes_read" => 0, + "url_mapping" => null, + ], // SQL output mode (file, stdout, mysql) — persisted for resume "sql_output" => null, // MySQL connection parameters — persisted for resume (password excluded) @@ -5992,6 +6530,12 @@ private function normalize_state(array $state): array $index_db, ); $state["db_index"] = $index_db; + $apply = $state["apply"] ?? []; + if (!is_array($apply)) { + $apply = []; + } + $apply = array_intersect_key($apply, $defaults["apply"]); + $state["apply"] = array_merge($defaults["apply"], $apply); return $state; } @@ -6576,6 +7120,29 @@ class StreamingContext "Output files:\n" . " db-tables.jsonl One JSON object per table\n", ], + "db-apply" => [ + "short" => "Apply SQL dump to a target MySQL database with URL rewriting", + "detail" => + "Reads /db.sql, optionally rewrites URLs, and executes\n" . + "all statements against a target MySQL database. Resumable.\n" . + "\n" . + "The parameter is kept for CLI consistency but ignored.\n" . + "\n" . + "Options:\n" . + " --target-host=HOST Target MySQL host (default: 127.0.0.1)\n" . + " --target-port=PORT Target MySQL port (default: 3306)\n" . + " --target-user=USER Target MySQL user (required)\n" . + " --target-pass=PASS Target MySQL password\n" . + " --target-db=NAME Target MySQL database (required)\n" . + " --url-mapping=FROM::TO URL mapping (repeatable)\n" . + " --abort Clear state and exit\n" . + " --verbose, -v Show detailed logs\n" . + "\n" . + "Example:\n" . + " php import.php db-apply - /path/to/import \\\n" . + " --target-user=root --target-db=wp_new \\\n" . + " --url-mapping=https://old.com::https://new.com\n", + ], "preflight" => [ "short" => "Run preflight check and print the full result as JSON", "detail" => @@ -6872,6 +7439,21 @@ class StreamingContext $options["mysql_password"] = substr($argv[$i], strlen("--mysql-password=")); } elseif (strpos($argv[$i], "--mysql-database=") === 0) { $options["mysql_database"] = substr($argv[$i], strlen("--mysql-database=")); + } elseif (strpos($argv[$i], "--target-host=") === 0) { + $options["target_host"] = substr($argv[$i], strlen("--target-host=")); + } elseif (strpos($argv[$i], "--target-port=") === 0) { + $options["target_port"] = (int) substr($argv[$i], strlen("--target-port=")); + } elseif (strpos($argv[$i], "--target-user=") === 0) { + $options["target_user"] = substr($argv[$i], strlen("--target-user=")); + } elseif (strpos($argv[$i], "--target-pass=") === 0) { + $options["target_pass"] = substr($argv[$i], strlen("--target-pass=")); + } elseif (strpos($argv[$i], "--target-db=") === 0) { + $options["target_db"] = substr($argv[$i], strlen("--target-db=")); + } elseif (strpos($argv[$i], "--url-mapping=") === 0) { + if (!isset($options["url_mapping"])) { + $options["url_mapping"] = []; + } + $options["url_mapping"][] = substr($argv[$i], strlen("--url-mapping=")); } else { fwrite(STDERR, "Unknown option: {$argv[$i]}\n"); exit(1); diff --git a/importer/lib/Base64ValueScanner.php b/importer/lib/Base64ValueScanner.php new file mode 100644 index 00000000..9ab84d8a --- /dev/null +++ b/importer/lib/Base64ValueScanner.php @@ -0,0 +1,122 @@ + int Position of the full expression in $sql + * 'length' => int Length of the full expression (including any CONVERT wrapper) + * 'value' => string The base64-decoded string value + * 'is_json' => bool Whether wrapped in CONVERT(... USING utf8mb4) + * + * @param string $sql The SQL statement to scan. + * @return array + */ + public static function scan(string $sql): array + { + $results = []; + $pos = 0; + $sql_len = strlen($sql); + + while ($pos < $sql_len) { + // Look for FROM_BASE64(' starting from current position + $fb_pos = strpos($sql, self::FROM_BASE64_PREFIX, $pos); + if ($fb_pos === false) { + break; + } + + // Find the closing ') + $payload_start = $fb_pos + self::FROM_BASE64_PREFIX_LEN; + $close_pos = strpos($sql, "')", $payload_start); + if ($close_pos === false) { + // Malformed — skip past this match + $pos = $payload_start; + continue; + } + + // Extract and decode the base64 payload + $b64_payload = substr($sql, $payload_start, $close_pos - $payload_start); + $decoded = base64_decode($b64_payload, true); + if ($decoded === false) { + // Invalid base64 — skip + $pos = $close_pos + 2; + continue; + } + + // Check if preceded by CONVERT( to detect JSON wrapper + $is_json = false; + $expr_offset = $fb_pos; + $expr_end = $close_pos + 2; // past the ') + + // CONVERT(FROM_BASE64('...') USING utf8mb4) + $convert_check_start = $fb_pos - 8; // strlen("CONVERT(") = 8 + if ( + $convert_check_start >= 0 && + substr($sql, $convert_check_start, 8) === "CONVERT(" + ) { + // Verify the USING utf8mb4) suffix follows + if ( + $expr_end + self::CONVERT_SUFFIX_LEN <= $sql_len && + substr($sql, $expr_end, self::CONVERT_SUFFIX_LEN) === self::CONVERT_SUFFIX + ) { + $is_json = true; + $expr_offset = $convert_check_start; + $expr_end = $expr_end + self::CONVERT_SUFFIX_LEN; + } + } + + $results[] = [ + 'offset' => $expr_offset, + 'length' => $expr_end - $expr_offset, + 'value' => $decoded, + 'is_json' => $is_json, + ]; + + $pos = $expr_end; + } + + return $results; + } + + /** + * Replace a base64 value at the given offset. + * + * Handles both FROM_BASE64('...') and CONVERT(FROM_BASE64('...') USING utf8mb4). + * Returns the modified SQL. + * + * @param string $sql The SQL statement. + * @param int $offset Start offset of the expression. + * @param int $length Length of the expression. + * @param string $new_value The new decoded value to encode. + * @param bool $is_json Whether to wrap in CONVERT(... USING utf8mb4). + * @return string The modified SQL. + */ + public static function replace(string $sql, int $offset, int $length, string $new_value, bool $is_json): string + { + $encoded = base64_encode($new_value); + + if ($is_json) { + $replacement = "CONVERT(FROM_BASE64('" . $encoded . "') USING utf8mb4)"; + } else { + $replacement = "FROM_BASE64('" . $encoded . "')"; + } + + return substr($sql, 0, $offset) . $replacement . substr($sql, $offset + $length); + } +} diff --git a/importer/lib/ContentClassifier.php b/importer/lib/ContentClassifier.php new file mode 100644 index 00000000..b8ef8399 --- /dev/null +++ b/importer/lib/ContentClassifier.php @@ -0,0 +1,99 @@ + Unique domains (scheme://host:port) */ + private array $domains = []; + + /** + * Scan a decoded string value for HTTP/HTTPS URLs and collect their domains. + * + * @param string $value The decoded database value to scan. + */ + public function scan(string $value): void + { + if ($value === '') { + return; + } + + $p = new URLInTextProcessor($value); + while ($p->next_url()) { + $parsed = $p->get_parsed_url(); + if (!$parsed) { + continue; + } + + $scheme = $parsed->protocol; // "https:" with colon + if ($scheme !== 'http:' && $scheme !== 'https:') { + continue; + } + + // Build origin: scheme://host[:port] + $scheme_clean = rtrim($scheme, ':'); + $host = $parsed->hostname; + $port = $parsed->port; + + $origin = $scheme_clean . '://' . $host; + if ($port !== '') { + $origin .= ':' . $port; + } + + $this->domains[$origin] = true; + } + } + + /** + * Get sorted list of unique domain origins. + * + * @return string[] Sorted list of domain origins. + */ + public function get_domains(): array + { + $domains = array_keys($this->domains); + sort($domains); + return $domains; + } + + /** + * Merge with a previously saved domain list. + * + * @param string[] $domains List of domain origins to merge. + */ + public function merge(array $domains): void + { + foreach ($domains as $domain) { + $this->domains[$domain] = true; + } + } +} diff --git a/importer/lib/SqlStatementRewriter.php b/importer/lib/SqlStatementRewriter.php new file mode 100644 index 00000000..47fe2de1 --- /dev/null +++ b/importer/lib/SqlStatementRewriter.php @@ -0,0 +1,68 @@ +url_rewriter = $url_rewriter; + } + + /** + * Rewrite URLs in a SQL statement. + * + * @param string $sql The SQL statement. + * @return string The modified SQL statement. + */ + public function rewrite(string $sql): string + { + // Quick check: if no base64 values, nothing to rewrite + if (strpos($sql, "FROM_BASE64(") === false) { + return $sql; + } + + // Scan for base64 values + $matches = Base64ValueScanner::scan($sql); + if (empty($matches)) { + return $sql; + } + + // Process in reverse offset order to preserve positions + $matches = array_reverse($matches); + + foreach ($matches as $match) { + $value = $match['value']; + $type = ContentClassifier::classify($value); + + // Skip serialized PHP + if ($type === ContentClassifier::TYPE_SERIALIZED_PHP) { + continue; + } + + // Rewrite URLs in the value + $rewritten = $this->url_rewriter->rewrite($value); + + // Only replace if the value actually changed + if ($rewritten !== $value) { + $sql = Base64ValueScanner::replace( + $sql, + $match['offset'], + $match['length'], + $rewritten, + $match['is_json'] + ); + } + } + + return $sql; + } +} diff --git a/importer/lib/SqlValueUrlRewriter.php b/importer/lib/SqlValueUrlRewriter.php new file mode 100644 index 00000000..54950b70 --- /dev/null +++ b/importer/lib/SqlValueUrlRewriter.php @@ -0,0 +1,158 @@ + URL mapping: source_url => target_url */ + private array $url_mapping; + + /** @var array Parsed mapping: [{from_url: URL, to_url: URL}] */ + private array $parsed_mapping; + + /** + * @param array $url_mapping Source URL => target URL mapping. + */ + public function __construct(array $url_mapping) + { + $this->url_mapping = $url_mapping; + $this->parsed_mapping = []; + foreach ($url_mapping as $from => $to) { + $this->parsed_mapping[] = [ + 'from_url' => WPURL::parse($from), + 'to_url' => WPURL::parse($to), + ]; + } + } + + /** + * Rewrite URLs in a single decoded value. + * + * @param string $value The decoded database value. + * @return string The rewritten value, or the original if no changes were made. + */ + public function rewrite(string $value): string + { + if ($value === '') { + return $value; + } + + $type = ContentClassifier::classify($value); + + switch ($type) { + case ContentClassifier::TYPE_SERIALIZED_PHP: + // Skip serialized PHP — rewriting would break s:N: length prefixes + return $value; + + case ContentClassifier::TYPE_JSON: + return $this->rewrite_json($value); + + case ContentClassifier::TYPE_TEXT: + default: + return $this->rewrite_text($value); + } + } + + /** + * Rewrite URLs in a JSON value by recursively walking all string values. + */ + private function rewrite_json(string $json): string + { + $data = json_decode($json, true); + if ($data === null && json_last_error() !== JSON_ERROR_NONE) { + return $json; + } + + $changed = false; + $data = $this->walk_json($data, $changed); + + if (!$changed) { + return $json; + } + + return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); + } + + /** + * Recursively walk a JSON-decoded structure and rewrite URLs in string values. + * + * @param mixed $data The JSON-decoded data. + * @param bool $changed Set to true if any value was changed. + * @return mixed The walked data with URLs rewritten. + */ + private function walk_json($data, bool &$changed) + { + if (is_string($data)) { + $rewritten = $this->rewrite_urls_in_text($data); + if ($rewritten !== $data) { + $changed = true; + return $rewritten; + } + return $data; + } + + if (is_array($data)) { + foreach ($data as $key => $value) { + $data[$key] = $this->walk_json($value, $changed); + } + } + + return $data; + } + + /** + * Rewrite URLs in text/HTML/block markup using wp_rewrite_urls(). + * This handles HTML attributes, block comment JSON, text nodes, and CSS url(). + */ + private function rewrite_text(string $value): string + { + return wp_rewrite_urls([ + 'block_markup' => $value, + 'url-mapping' => $this->url_mapping, + ]); + } + + /** + * Rewrite URLs in a plain text string using URLInTextProcessor. + * Always produces absolute URLs (not relative) since database values + * should contain complete URLs. + */ + private function rewrite_urls_in_text(string $text): string + { + $p = new URLInTextProcessor($text); + while ($p->next_url()) { + $parsed_url = $p->get_parsed_url(); + if (!$parsed_url) { + continue; + } + foreach ($this->parsed_mapping as $mapping) { + if (is_child_url_of($parsed_url, $mapping['from_url'])) { + $result = WPURL::replace_base_url($parsed_url, [ + 'old_base_url' => $mapping['from_url'], + 'new_base_url' => $mapping['to_url'], + ]); + if ($result !== false) { + $p->set_raw_url($result->new_raw_url); + } + break; + } + } + } + return $p->get_updated_text(); + } +} diff --git a/importer/lib/mysql-query-stream/class-wp-mysql-lexer.php b/importer/lib/mysql-query-stream/class-wp-mysql-lexer.php new file mode 100644 index 00000000..b9f582c3 --- /dev/null +++ b/importer/lib/mysql-query-stream/class-wp-mysql-lexer.php @@ -0,0 +1,2997 @@ +sql_modes". + * The list of the SQL modes is not exhaustive. Only the ones that influence + * the lexer behavior are included in this list. + * + * See: + * https://dev.mysql.com/doc/refman/8.4/en/sql-mode.html + */ + const SQL_MODE_HIGH_NOT_PRECEDENCE = 1; + const SQL_MODE_PIPES_AS_CONCAT = 2; + const SQL_MODE_IGNORE_SPACE = 4; + const SQL_MODE_NO_BACKSLASH_ESCAPES = 8; + + /** + * Character masks for frequently used character classes. + * + * These are intended to be used with "strspn()" and "strcspn()" functions + * for fast character class matching in the SQL payload. + */ + const WHITESPACE_MASK = " \t\n\r\f"; + const DIGIT_MASK = '0123456789'; + const HEX_DIGIT_MASK = '0123456789abcdefABCDEF'; + + /** + * Tokens from the MySQL Workbench "predefined.tokens" list. + * + * This list preserves the token names and IDs from the MySQL Workbench + * "predefined.tokens" list, adding some tokens missing from the list. + * + * See: + * https://github.com/mysql/mysql-workbench/blob/8.0.38/library/parsers/grammars/predefined.tokens + */ + const ACCESSIBLE_SYMBOL = 1; + const ACCOUNT_SYMBOL = 2; + const ACTION_SYMBOL = 3; + const ADD_SYMBOL = 4; + const ADDDATE_SYMBOL = 5; + const AFTER_SYMBOL = 6; + const AGAINST_SYMBOL = 7; + const AGGREGATE_SYMBOL = 8; + const ALGORITHM_SYMBOL = 9; + const ALL_SYMBOL = 10; + const ALTER_SYMBOL = 11; + const ALWAYS_SYMBOL = 12; + const ANALYSE_SYMBOL = 13; + const ANALYZE_SYMBOL = 14; + const AND_SYMBOL = 15; + const ANY_SYMBOL = 16; + const AS_SYMBOL = 17; + const ASC_SYMBOL = 18; + const ASCII_SYMBOL = 19; + const ASENSITIVE_SYMBOL = 20; + const AT_SYMBOL = 21; + const AUTHORS_SYMBOL = 22; + const AUTOEXTEND_SIZE_SYMBOL = 23; + const AUTO_INCREMENT_SYMBOL = 24; + const AVG_ROW_LENGTH_SYMBOL = 25; + const AVG_SYMBOL = 26; + const BACKUP_SYMBOL = 27; + const BEFORE_SYMBOL = 28; + const BEGIN_SYMBOL = 29; + const BETWEEN_SYMBOL = 30; + const BIGINT_SYMBOL = 31; + const BINARY_SYMBOL = 32; + const BINLOG_SYMBOL = 33; + const BIN_NUM_SYMBOL = 34; + const BIT_AND_SYMBOL = 35; + const BIT_OR_SYMBOL = 36; + const BIT_SYMBOL = 37; + const BIT_XOR_SYMBOL = 38; + const BLOB_SYMBOL = 39; + const BLOCK_SYMBOL = 40; + const BOOLEAN_SYMBOL = 41; + const BOOL_SYMBOL = 42; + const BOTH_SYMBOL = 43; + const BTREE_SYMBOL = 44; + const BY_SYMBOL = 45; + const BYTE_SYMBOL = 46; + const CACHE_SYMBOL = 47; + const CALL_SYMBOL = 48; + const CASCADE_SYMBOL = 49; + const CASCADED_SYMBOL = 50; + const CASE_SYMBOL = 51; + const CAST_SYMBOL = 52; + const CATALOG_NAME_SYMBOL = 53; + const CHAIN_SYMBOL = 54; + const CHANGE_SYMBOL = 55; + const CHANGED_SYMBOL = 56; + const CHANNEL_SYMBOL = 57; + const CHARSET_SYMBOL = 58; + const CHARACTER_SYMBOL = 59; + const CHAR_SYMBOL = 60; + const CHECKSUM_SYMBOL = 61; + const CHECK_SYMBOL = 62; + const CIPHER_SYMBOL = 63; + const CLASS_ORIGIN_SYMBOL = 64; + const CLIENT_SYMBOL = 65; + const CLOSE_SYMBOL = 66; + const COALESCE_SYMBOL = 67; + const CODE_SYMBOL = 68; + const COLLATE_SYMBOL = 69; + const COLLATION_SYMBOL = 70; + const COLUMNS_SYMBOL = 71; + const COLUMN_SYMBOL = 72; + const COLUMN_NAME_SYMBOL = 73; + const COLUMN_FORMAT_SYMBOL = 74; + const COMMENT_SYMBOL = 75; + const COMMITTED_SYMBOL = 76; + const COMMIT_SYMBOL = 77; + const COMPACT_SYMBOL = 78; + const COMPLETION_SYMBOL = 79; + const COMPRESSED_SYMBOL = 80; + const COMPRESSION_SYMBOL = 81; + const CONCURRENT_SYMBOL = 82; + const CONDITION_SYMBOL = 83; + const CONNECTION_SYMBOL = 84; + const CONSISTENT_SYMBOL = 85; + const CONSTRAINT_SYMBOL = 86; + const CONSTRAINT_CATALOG_SYMBOL = 87; + const CONSTRAINT_NAME_SYMBOL = 88; + const CONSTRAINT_SCHEMA_SYMBOL = 89; + const CONTAINS_SYMBOL = 90; + const CONTEXT_SYMBOL = 91; + const CONTINUE_SYMBOL = 92; + const CONTRIBUTORS_SYMBOL = 93; + const CONVERT_SYMBOL = 94; + const COUNT_SYMBOL = 95; + const CPU_SYMBOL = 96; + const CREATE_SYMBOL = 97; + const CROSS_SYMBOL = 98; + const CUBE_SYMBOL = 99; + const CURDATE_SYMBOL = 100; + const CURRENT_SYMBOL = 101; + const CURRENT_DATE_SYMBOL = 102; + const CURRENT_TIME_SYMBOL = 103; + const CURRENT_TIMESTAMP_SYMBOL = 104; + const CURRENT_USER_SYMBOL = 105; + const CURSOR_SYMBOL = 106; + const CURSOR_NAME_SYMBOL = 107; + const CURTIME_SYMBOL = 108; + const DATABASE_SYMBOL = 109; + const DATABASES_SYMBOL = 110; + const DATAFILE_SYMBOL = 111; + const DATA_SYMBOL = 112; + const DATETIME_SYMBOL = 113; + const DATE_ADD_SYMBOL = 114; + const DATE_SUB_SYMBOL = 115; + const DATE_SYMBOL = 116; + const DAYOFMONTH_SYMBOL = 117; + const DAY_HOUR_SYMBOL = 118; + const DAY_MICROSECOND_SYMBOL = 119; + const DAY_MINUTE_SYMBOL = 120; + const DAY_SECOND_SYMBOL = 121; + const DAY_SYMBOL = 122; + const DEALLOCATE_SYMBOL = 123; + const DEC_SYMBOL = 124; + const DECIMAL_NUM_SYMBOL = 125; + const DECIMAL_SYMBOL = 126; + const DECLARE_SYMBOL = 127; + const DEFAULT_SYMBOL = 128; + const DEFAULT_AUTH_SYMBOL = 129; + const DEFINER_SYMBOL = 130; + const DELAYED_SYMBOL = 131; + const DELAY_KEY_WRITE_SYMBOL = 132; + const DELETE_SYMBOL = 133; + const DESC_SYMBOL = 134; + const DESCRIBE_SYMBOL = 135; + const DES_KEY_FILE_SYMBOL = 136; + const DETERMINISTIC_SYMBOL = 137; + const DIAGNOSTICS_SYMBOL = 138; + const DIRECTORY_SYMBOL = 139; + const DISABLE_SYMBOL = 140; + const DISCARD_SYMBOL = 141; + const DISK_SYMBOL = 142; + const DISTINCT_SYMBOL = 143; + const DISTINCTROW_SYMBOL = 144; + const DIV_SYMBOL = 145; + const DOUBLE_SYMBOL = 146; + const DO_SYMBOL = 147; + const DROP_SYMBOL = 148; + const DUAL_SYMBOL = 149; + const DUMPFILE_SYMBOL = 150; + const DUPLICATE_SYMBOL = 151; + const DYNAMIC_SYMBOL = 152; + const EACH_SYMBOL = 153; + const ELSE_SYMBOL = 154; + const ELSEIF_SYMBOL = 155; + const ENABLE_SYMBOL = 156; + const ENCLOSED_SYMBOL = 157; + const ENCRYPTION_SYMBOL = 158; + const END_SYMBOL = 159; + const ENDS_SYMBOL = 160; + const END_OF_INPUT_SYMBOL = 161; // defined in "predefined.tokens", but not used + const ENGINES_SYMBOL = 162; + const ENGINE_SYMBOL = 163; + const ENUM_SYMBOL = 164; + const ERROR_SYMBOL = 165; + const ERRORS_SYMBOL = 166; + const ESCAPED_SYMBOL = 167; + const ESCAPE_SYMBOL = 168; + const EVENTS_SYMBOL = 169; + const EVENT_SYMBOL = 170; + const EVERY_SYMBOL = 171; + const EXCHANGE_SYMBOL = 172; + const EXECUTE_SYMBOL = 173; + const EXISTS_SYMBOL = 174; + const EXIT_SYMBOL = 175; + const EXPANSION_SYMBOL = 176; + const EXPIRE_SYMBOL = 177; + const EXPLAIN_SYMBOL = 178; + const EXPORT_SYMBOL = 179; + const EXTENDED_SYMBOL = 180; + const EXTENT_SIZE_SYMBOL = 181; + const EXTRACT_SYMBOL = 182; + const FALSE_SYMBOL = 183; + const FAST_SYMBOL = 184; + const FAULTS_SYMBOL = 185; + const FETCH_SYMBOL = 186; + const FIELDS_SYMBOL = 187; + const FILE_SYMBOL = 188; + const FILE_BLOCK_SIZE_SYMBOL = 189; + const FILTER_SYMBOL = 190; + const FIRST_SYMBOL = 191; + const FIXED_SYMBOL = 192; + const FLOAT4_SYMBOL = 193; + const FLOAT8_SYMBOL = 194; + const FLOAT_SYMBOL = 195; + const FLUSH_SYMBOL = 196; + const FOLLOWS_SYMBOL = 197; + const FORCE_SYMBOL = 198; + const FOREIGN_SYMBOL = 199; + const FOR_SYMBOL = 200; + const FORMAT_SYMBOL = 201; + const FOUND_SYMBOL = 202; + const FROM_SYMBOL = 203; + const FULL_SYMBOL = 204; + const FULLTEXT_SYMBOL = 205; + const FUNCTION_SYMBOL = 206; + const GET_SYMBOL = 207; + const GENERAL_SYMBOL = 208; + const GENERATED_SYMBOL = 209; + const GROUP_REPLICATION_SYMBOL = 210; + const GEOMETRYCOLLECTION_SYMBOL = 211; + const GEOMETRY_SYMBOL = 212; + const GET_FORMAT_SYMBOL = 213; + const GLOBAL_SYMBOL = 214; + const GRANT_SYMBOL = 215; + const GRANTS_SYMBOL = 216; + const GROUP_SYMBOL = 217; + const GROUP_CONCAT_SYMBOL = 218; + const HANDLER_SYMBOL = 219; + const HASH_SYMBOL = 220; + const HAVING_SYMBOL = 221; + const HELP_SYMBOL = 222; + const HIGH_PRIORITY_SYMBOL = 223; + const HOST_SYMBOL = 224; + const HOSTS_SYMBOL = 225; + const HOUR_MICROSECOND_SYMBOL = 226; + const HOUR_MINUTE_SYMBOL = 227; + const HOUR_SECOND_SYMBOL = 228; + const HOUR_SYMBOL = 229; + const IDENTIFIED_SYMBOL = 230; + const IF_SYMBOL = 231; + const IGNORE_SYMBOL = 232; + const IGNORE_SERVER_IDS_SYMBOL = 233; + const IMPORT_SYMBOL = 234; + const INDEXES_SYMBOL = 235; + const INDEX_SYMBOL = 236; + const INFILE_SYMBOL = 237; + const INITIAL_SIZE_SYMBOL = 238; + const INNER_SYMBOL = 239; + const INOUT_SYMBOL = 240; + const INSENSITIVE_SYMBOL = 241; + const INSERT_SYMBOL = 242; + const INSERT_METHOD_SYMBOL = 243; + const INSTANCE_SYMBOL = 244; + const INSTALL_SYMBOL = 245; + const INTEGER_SYMBOL = 246; + const INTERVAL_SYMBOL = 247; + const INTO_SYMBOL = 248; + const INT_SYMBOL = 249; + const INVOKER_SYMBOL = 250; + const IN_SYMBOL = 251; + const IO_AFTER_GTIDS_SYMBOL = 252; + const IO_BEFORE_GTIDS_SYMBOL = 253; + const IO_THREAD_SYMBOL = 254; + const IO_SYMBOL = 255; + const IPC_SYMBOL = 256; + const IS_SYMBOL = 257; + const ISOLATION_SYMBOL = 258; + const ISSUER_SYMBOL = 259; + const ITERATE_SYMBOL = 260; + const JOIN_SYMBOL = 261; + const JSON_SYMBOL = 262; + const KEYS_SYMBOL = 263; + const KEY_BLOCK_SIZE_SYMBOL = 264; + const KEY_SYMBOL = 265; + const KILL_SYMBOL = 266; + const LANGUAGE_SYMBOL = 267; + const LAST_SYMBOL = 268; + const LEADING_SYMBOL = 269; + const LEAVES_SYMBOL = 270; + const LEAVE_SYMBOL = 271; + const LEFT_SYMBOL = 272; + const LESS_SYMBOL = 273; + const LEVEL_SYMBOL = 274; + const LIKE_SYMBOL = 275; + const LIMIT_SYMBOL = 276; + const LINEAR_SYMBOL = 277; + const LINES_SYMBOL = 278; + const LINESTRING_SYMBOL = 279; + const LIST_SYMBOL = 280; + const LOAD_SYMBOL = 281; + const LOCALTIME_SYMBOL = 282; + const LOCALTIMESTAMP_SYMBOL = 283; + const LOCAL_SYMBOL = 284; + const LOCATOR_SYMBOL = 285; + const LOCKS_SYMBOL = 286; + const LOCK_SYMBOL = 287; + const LOGFILE_SYMBOL = 288; + const LOGS_SYMBOL = 289; + const LONGBLOB_SYMBOL = 290; + const LONGTEXT_SYMBOL = 291; + const LONG_NUM_SYMBOL = 292; + const LONG_SYMBOL = 293; + const LOOP_SYMBOL = 294; + const LOW_PRIORITY_SYMBOL = 295; + const MASTER_AUTO_POSITION_SYMBOL = 296; + const MASTER_BIND_SYMBOL = 297; + const MASTER_CONNECT_RETRY_SYMBOL = 298; + const MASTER_DELAY_SYMBOL = 299; + const MASTER_HOST_SYMBOL = 300; + const MASTER_LOG_FILE_SYMBOL = 301; + const MASTER_LOG_POS_SYMBOL = 302; + const MASTER_PASSWORD_SYMBOL = 303; + const MASTER_PORT_SYMBOL = 304; + const MASTER_RETRY_COUNT_SYMBOL = 305; + const MASTER_SERVER_ID_SYMBOL = 306; + const MASTER_SSL_CAPATH_SYMBOL = 307; + const MASTER_SSL_CA_SYMBOL = 308; + const MASTER_SSL_CERT_SYMBOL = 309; + const MASTER_SSL_CIPHER_SYMBOL = 310; + const MASTER_SSL_CRL_SYMBOL = 311; + const MASTER_SSL_CRLPATH_SYMBOL = 312; + const MASTER_SSL_KEY_SYMBOL = 313; + const MASTER_SSL_SYMBOL = 314; + const MASTER_SSL_VERIFY_SERVER_CERT_SYMBOL = 315; + const MASTER_SYMBOL = 316; + const MASTER_TLS_VERSION_SYMBOL = 317; + const MASTER_USER_SYMBOL = 318; + const MASTER_HEARTBEAT_PERIOD_SYMBOL = 319; + const MATCH_SYMBOL = 320; + const MAX_CONNECTIONS_PER_HOUR_SYMBOL = 321; + const MAX_QUERIES_PER_HOUR_SYMBOL = 322; + const MAX_ROWS_SYMBOL = 323; + const MAX_SIZE_SYMBOL = 324; + const MAX_STATEMENT_TIME_SYMBOL = 325; + const MAX_SYMBOL = 326; + const MAX_UPDATES_PER_HOUR_SYMBOL = 327; + const MAX_USER_CONNECTIONS_SYMBOL = 328; + const MAXVALUE_SYMBOL = 329; + const MEDIUMBLOB_SYMBOL = 330; + const MEDIUMINT_SYMBOL = 331; + const MEDIUMTEXT_SYMBOL = 332; + const MEDIUM_SYMBOL = 333; + const MEMORY_SYMBOL = 334; + const MERGE_SYMBOL = 335; + const MESSAGE_TEXT_SYMBOL = 336; + const MICROSECOND_SYMBOL = 337; + const MID_SYMBOL = 338; + const MIDDLEINT_SYMBOL = 339; + const MIGRATE_SYMBOL = 340; + const MINUTE_MICROSECOND_SYMBOL = 341; + const MINUTE_SECOND_SYMBOL = 342; + const MINUTE_SYMBOL = 343; + const MIN_ROWS_SYMBOL = 344; + const MIN_SYMBOL = 345; + const MODE_SYMBOL = 346; + const MODIFIES_SYMBOL = 347; + const MODIFY_SYMBOL = 348; + const MOD_SYMBOL = 349; + const MONTH_SYMBOL = 350; + const MULTILINESTRING_SYMBOL = 351; + const MULTIPOINT_SYMBOL = 352; + const MULTIPOLYGON_SYMBOL = 353; + const MUTEX_SYMBOL = 354; + const MYSQL_ERRNO_SYMBOL = 355; + const NAMES_SYMBOL = 356; + const NAME_SYMBOL = 357; + const NATIONAL_SYMBOL = 358; + const NATURAL_SYMBOL = 359; + const NCHAR_STRING_SYMBOL = 360; + const NCHAR_SYMBOL = 361; + const NDB_SYMBOL = 362; + const NDBCLUSTER_SYMBOL = 363; + const NEG_SYMBOL = 364; + const NEVER_SYMBOL = 365; + const NEW_SYMBOL = 366; + const NEXT_SYMBOL = 367; + const NODEGROUP_SYMBOL = 368; + const NONE_SYMBOL = 369; + const NONBLOCKING_SYMBOL = 370; + const NOT_SYMBOL = 371; + const NOW_SYMBOL = 372; + const NO_SYMBOL = 373; + const NO_WAIT_SYMBOL = 374; + const NO_WRITE_TO_BINLOG_SYMBOL = 375; + const NULL_SYMBOL = 376; + const NUMBER_SYMBOL = 377; + const NUMERIC_SYMBOL = 378; + const NVARCHAR_SYMBOL = 379; + const OFFLINE_SYMBOL = 380; + const OFFSET_SYMBOL = 381; + const OLD_PASSWORD_SYMBOL = 382; + const ON_SYMBOL = 383; + const ONE_SYMBOL = 384; + const ONLINE_SYMBOL = 385; + const ONLY_SYMBOL = 386; + const OPEN_SYMBOL = 387; + const OPTIMIZE_SYMBOL = 388; + const OPTIMIZER_COSTS_SYMBOL = 389; + const OPTIONS_SYMBOL = 390; + const OPTION_SYMBOL = 391; + const OPTIONALLY_SYMBOL = 392; + const ORDER_SYMBOL = 393; + const OR_SYMBOL = 394; + const OUTER_SYMBOL = 395; + const OUTFILE_SYMBOL = 396; + const OUT_SYMBOL = 397; + const OWNER_SYMBOL = 398; + const PACK_KEYS_SYMBOL = 399; + const PAGE_SYMBOL = 400; + const PARSER_SYMBOL = 401; + const PARTIAL_SYMBOL = 402; + const PARTITIONING_SYMBOL = 403; + const PARTITIONS_SYMBOL = 404; + const PARTITION_SYMBOL = 405; + const PASSWORD_SYMBOL = 406; + const PHASE_SYMBOL = 407; + const PLUGINS_SYMBOL = 408; + const PLUGIN_DIR_SYMBOL = 409; + const PLUGIN_SYMBOL = 410; + const POINT_SYMBOL = 411; + const POLYGON_SYMBOL = 412; + const PORT_SYMBOL = 413; + const POSITION_SYMBOL = 414; + const PRECEDES_SYMBOL = 415; + const PRECISION_SYMBOL = 416; + const PREPARE_SYMBOL = 417; + const PRESERVE_SYMBOL = 418; + const PREV_SYMBOL = 419; + const PRIMARY_SYMBOL = 420; + const PRIVILEGES_SYMBOL = 421; + const PROCEDURE_SYMBOL = 422; + const PROCESS_SYMBOL = 423; + const PROCESSLIST_SYMBOL = 424; + const PROFILE_SYMBOL = 425; + const PROFILES_SYMBOL = 426; + const PROXY_SYMBOL = 427; + const PURGE_SYMBOL = 428; + const QUARTER_SYMBOL = 429; + const QUERY_SYMBOL = 430; + const QUICK_SYMBOL = 431; + const RANGE_SYMBOL = 432; + const READS_SYMBOL = 433; + const READ_ONLY_SYMBOL = 434; + const READ_SYMBOL = 435; + const READ_WRITE_SYMBOL = 436; + const REAL_SYMBOL = 437; + const REBUILD_SYMBOL = 438; + const RECOVER_SYMBOL = 439; + const REDOFILE_SYMBOL = 440; + const REDO_BUFFER_SIZE_SYMBOL = 441; + const REDUNDANT_SYMBOL = 442; + const REFERENCES_SYMBOL = 443; + const REGEXP_SYMBOL = 444; + const RELAY_SYMBOL = 445; + const RELAYLOG_SYMBOL = 446; + const RELAY_LOG_FILE_SYMBOL = 447; + const RELAY_LOG_POS_SYMBOL = 448; + const RELAY_THREAD_SYMBOL = 449; + const RELEASE_SYMBOL = 450; + const RELOAD_SYMBOL = 451; + const REMOVE_SYMBOL = 452; + const RENAME_SYMBOL = 453; + const REORGANIZE_SYMBOL = 454; + const REPAIR_SYMBOL = 455; + const REPEATABLE_SYMBOL = 456; + const REPEAT_SYMBOL = 457; + const REPLACE_SYMBOL = 458; + const REPLICATION_SYMBOL = 459; + const REPLICATE_DO_DB_SYMBOL = 460; + const REPLICATE_IGNORE_DB_SYMBOL = 461; + const REPLICATE_DO_TABLE_SYMBOL = 462; + const REPLICATE_IGNORE_TABLE_SYMBOL = 463; + const REPLICATE_WILD_DO_TABLE_SYMBOL = 464; + const REPLICATE_WILD_IGNORE_TABLE_SYMBOL = 465; + const REPLICATE_REWRITE_DB_SYMBOL = 466; + const REQUIRE_SYMBOL = 467; + const RESET_SYMBOL = 468; + const RESIGNAL_SYMBOL = 469; + const RESTORE_SYMBOL = 470; + const RESTRICT_SYMBOL = 471; + const RESUME_SYMBOL = 472; + const RETURNED_SQLSTATE_SYMBOL = 473; + const RETURNS_SYMBOL = 474; + const RETURN_SYMBOL = 475; + const REVERSE_SYMBOL = 476; + const REVOKE_SYMBOL = 477; + const RIGHT_SYMBOL = 478; + const RLIKE_SYMBOL = 479; + const ROLLBACK_SYMBOL = 480; + const ROLLUP_SYMBOL = 481; + const ROTATE_SYMBOL = 482; + const ROUTINE_SYMBOL = 483; + const ROWS_SYMBOL = 484; + const ROW_COUNT_SYMBOL = 485; + const ROW_FORMAT_SYMBOL = 486; + const ROW_SYMBOL = 487; + const RTREE_SYMBOL = 488; + const SAVEPOINT_SYMBOL = 489; + const SCHEDULE_SYMBOL = 490; + const SCHEMA_SYMBOL = 491; + const SCHEMA_NAME_SYMBOL = 492; + const SCHEMAS_SYMBOL = 493; + const SECOND_MICROSECOND_SYMBOL = 494; + const SECOND_SYMBOL = 495; + const SECURITY_SYMBOL = 496; + const SELECT_SYMBOL = 497; + const SENSITIVE_SYMBOL = 498; + const SEPARATOR_SYMBOL = 499; + const SERIALIZABLE_SYMBOL = 500; + const SERIAL_SYMBOL = 501; + const SESSION_SYMBOL = 502; + const SERVER_SYMBOL = 503; + const SERVER_OPTIONS_SYMBOL = 504; + const SESSION_USER_SYMBOL = 505; + const SET_SYMBOL = 506; + const SET_VAR_SYMBOL = 507; + const SHARE_SYMBOL = 508; + const SHOW_SYMBOL = 509; + const SHUTDOWN_SYMBOL = 510; + const SIGNAL_SYMBOL = 511; + const SIGNED_SYMBOL = 512; + const SIMPLE_SYMBOL = 513; + const SLAVE_SYMBOL = 514; + const SLOW_SYMBOL = 515; + const SMALLINT_SYMBOL = 516; + const SNAPSHOT_SYMBOL = 517; + const SOME_SYMBOL = 518; + const SOCKET_SYMBOL = 519; + const SONAME_SYMBOL = 520; + const SOUNDS_SYMBOL = 521; + const SOURCE_SYMBOL = 522; + const SPATIAL_SYMBOL = 523; + const SPECIFIC_SYMBOL = 524; + const SQLEXCEPTION_SYMBOL = 525; + const SQLSTATE_SYMBOL = 526; + const SQLWARNING_SYMBOL = 527; + const SQL_AFTER_GTIDS_SYMBOL = 528; + const SQL_AFTER_MTS_GAPS_SYMBOL = 529; + const SQL_BEFORE_GTIDS_SYMBOL = 530; + const SQL_BIG_RESULT_SYMBOL = 531; + const SQL_BUFFER_RESULT_SYMBOL = 532; + const SQL_CACHE_SYMBOL = 533; + const SQL_CALC_FOUND_ROWS_SYMBOL = 534; + const SQL_NO_CACHE_SYMBOL = 535; + const SQL_SMALL_RESULT_SYMBOL = 536; + const SQL_SYMBOL = 537; + const SQL_THREAD_SYMBOL = 538; + const SSL_SYMBOL = 539; + const STACKED_SYMBOL = 540; + const STARTING_SYMBOL = 541; + const STARTS_SYMBOL = 542; + const START_SYMBOL = 543; + const STATS_AUTO_RECALC_SYMBOL = 544; + const STATS_PERSISTENT_SYMBOL = 545; + const STATS_SAMPLE_PAGES_SYMBOL = 546; + const STATUS_SYMBOL = 547; + const STDDEV_SAMP_SYMBOL = 548; + const STDDEV_SYMBOL = 549; + const STDDEV_POP_SYMBOL = 550; + const STD_SYMBOL = 551; + const STOP_SYMBOL = 552; + const STORAGE_SYMBOL = 553; + const STORED_SYMBOL = 554; + const STRAIGHT_JOIN_SYMBOL = 555; + const STRING_SYMBOL = 556; + const SUBCLASS_ORIGIN_SYMBOL = 557; + const SUBDATE_SYMBOL = 558; + const SUBJECT_SYMBOL = 559; + const SUBPARTITIONS_SYMBOL = 560; + const SUBPARTITION_SYMBOL = 561; + const SUBSTR_SYMBOL = 562; + const SUBSTRING_SYMBOL = 563; + const SUM_SYMBOL = 564; + const SUPER_SYMBOL = 565; + const SUSPEND_SYMBOL = 566; + const SWAPS_SYMBOL = 567; + const SWITCHES_SYMBOL = 568; + const SYSDATE_SYMBOL = 569; + const SYSTEM_USER_SYMBOL = 570; + const TABLES_SYMBOL = 571; + const TABLESPACE_SYMBOL = 572; + const TABLE_REF_PRIORITY_SYMBOL = 573; + const TABLE_SYMBOL = 574; + const TABLE_CHECKSUM_SYMBOL = 575; + const TABLE_NAME_SYMBOL = 576; + const TEMPORARY_SYMBOL = 577; + const TEMPTABLE_SYMBOL = 578; + const TERMINATED_SYMBOL = 579; + const TEXT_SYMBOL = 580; + const THAN_SYMBOL = 581; + const THEN_SYMBOL = 582; + const TIMESTAMP_SYMBOL = 583; + const TIMESTAMP_ADD_SYMBOL = 584; + const TIMESTAMP_DIFF_SYMBOL = 585; + const TIME_SYMBOL = 586; + const TINYBLOB_SYMBOL = 587; + const TINYINT_SYMBOL = 588; + const TINYTEXT_SYMBOL = 589; + const TO_SYMBOL = 590; + const TRAILING_SYMBOL = 591; + const TRANSACTION_SYMBOL = 592; + const TRIGGERS_SYMBOL = 593; + const TRIGGER_SYMBOL = 594; + const TRIM_SYMBOL = 595; + const TRUE_SYMBOL = 596; + const TRUNCATE_SYMBOL = 597; + const TYPES_SYMBOL = 598; + const TYPE_SYMBOL = 599; + const UDF_RETURNS_SYMBOL = 600; + const UNCOMMITTED_SYMBOL = 601; + const UNDEFINED_SYMBOL = 602; + const UNDOFILE_SYMBOL = 603; + const UNDO_BUFFER_SIZE_SYMBOL = 604; + const UNDO_SYMBOL = 605; + const UNICODE_SYMBOL = 606; + const UNINSTALL_SYMBOL = 607; + const UNION_SYMBOL = 608; + const UNIQUE_SYMBOL = 609; + const UNKNOWN_SYMBOL = 610; + const UNLOCK_SYMBOL = 611; + const UNSIGNED_SYMBOL = 612; + const UNTIL_SYMBOL = 613; + const UPDATE_SYMBOL = 614; + const UPGRADE_SYMBOL = 615; + const USAGE_SYMBOL = 616; + const USER_RESOURCES_SYMBOL = 617; + const USER_SYMBOL = 618; + const USE_FRM_SYMBOL = 619; + const USE_SYMBOL = 620; + const USING_SYMBOL = 621; + const UTC_DATE_SYMBOL = 622; + const UTC_TIMESTAMP_SYMBOL = 623; + const UTC_TIME_SYMBOL = 624; + const VALIDATION_SYMBOL = 625; + const VALUES_SYMBOL = 626; + const VALUE_SYMBOL = 627; + const VARBINARY_SYMBOL = 628; + const VARCHAR_SYMBOL = 629; + const VARCHARACTER_SYMBOL = 630; + const VARIABLES_SYMBOL = 631; + const VARIANCE_SYMBOL = 632; + const VARYING_SYMBOL = 633; + const VAR_POP_SYMBOL = 634; + const VAR_SAMP_SYMBOL = 635; + const VIEW_SYMBOL = 636; + const VIRTUAL_SYMBOL = 637; + const WAIT_SYMBOL = 638; + const WARNINGS_SYMBOL = 639; + const WEEK_SYMBOL = 640; + const WEIGHT_STRING_SYMBOL = 641; + const WHEN_SYMBOL = 642; + const WHERE_SYMBOL = 643; + const WHILE_SYMBOL = 644; + const WITH_SYMBOL = 645; + const WITHOUT_SYMBOL = 646; + const WORK_SYMBOL = 647; + const WRAPPER_SYMBOL = 648; + const WRITE_SYMBOL = 649; + const X509_SYMBOL = 650; + const XA_SYMBOL = 651; + const XID_SYMBOL = 652; + const XML_SYMBOL = 653; + const XOR_SYMBOL = 654; + const YEAR_MONTH_SYMBOL = 655; + const YEAR_SYMBOL = 656; + const ZEROFILL_SYMBOL = 657; + const PERSIST_SYMBOL = 658; + const ROLE_SYMBOL = 659; + const ADMIN_SYMBOL = 660; + const INVISIBLE_SYMBOL = 661; + const VISIBLE_SYMBOL = 662; + const EXCEPT_SYMBOL = 663; + const COMPONENT_SYMBOL = 664; + const RECURSIVE_SYMBOL = 665; + const JSON_OBJECTAGG_SYMBOL = 666; + const JSON_ARRAYAGG_SYMBOL = 667; + const OF_SYMBOL = 668; + const SKIP_SYMBOL = 669; + const LOCKED_SYMBOL = 670; + const NOWAIT_SYMBOL = 671; + const GROUPING_SYMBOL = 672; + const PERSIST_ONLY_SYMBOL = 673; + const HISTOGRAM_SYMBOL = 674; + const BUCKETS_SYMBOL = 675; + const REMOTE_SYMBOL = 676; + const CLONE_SYMBOL = 677; + const CUME_DIST_SYMBOL = 678; + const DENSE_RANK_SYMBOL = 679; + const EXCLUDE_SYMBOL = 680; + const FIRST_VALUE_SYMBOL = 681; + const FOLLOWING_SYMBOL = 682; + const GROUPS_SYMBOL = 683; + const LAG_SYMBOL = 684; + const LAST_VALUE_SYMBOL = 685; + const LEAD_SYMBOL = 686; + const NTH_VALUE_SYMBOL = 687; + const NTILE_SYMBOL = 688; + const NULLS_SYMBOL = 689; + const OTHERS_SYMBOL = 690; + const OVER_SYMBOL = 691; + const PERCENT_RANK_SYMBOL = 692; + const PRECEDING_SYMBOL = 693; + const RANK_SYMBOL = 694; + const RESPECT_SYMBOL = 695; + const ROW_NUMBER_SYMBOL = 696; + const TIES_SYMBOL = 697; + const UNBOUNDED_SYMBOL = 698; + const WINDOW_SYMBOL = 699; + const EMPTY_SYMBOL = 700; + const JSON_TABLE_SYMBOL = 701; + const NESTED_SYMBOL = 702; + const ORDINALITY_SYMBOL = 703; + const PATH_SYMBOL = 704; + const HISTORY_SYMBOL = 705; + const REUSE_SYMBOL = 706; + const SRID_SYMBOL = 707; + const THREAD_PRIORITY_SYMBOL = 708; + const RESOURCE_SYMBOL = 709; + const SYSTEM_SYMBOL = 710; + const VCPU_SYMBOL = 711; + const MASTER_PUBLIC_KEY_PATH_SYMBOL = 712; + const GET_MASTER_PUBLIC_KEY_SYMBOL = 713; + const RESTART_SYMBOL = 714; + const DEFINITION_SYMBOL = 715; + const DESCRIPTION_SYMBOL = 716; + const ORGANIZATION_SYMBOL = 717; + const REFERENCE_SYMBOL = 718; + const OPTIONAL_SYMBOL = 719; + const SECONDARY_SYMBOL = 720; + const SECONDARY_ENGINE_SYMBOL = 721; + const SECONDARY_LOAD_SYMBOL = 722; + const SECONDARY_UNLOAD_SYMBOL = 723; + const ACTIVE_SYMBOL = 724; + const INACTIVE_SYMBOL = 725; + const LATERAL_SYMBOL = 726; + const RETAIN_SYMBOL = 727; + const OLD_SYMBOL = 728; + const NETWORK_NAMESPACE_SYMBOL = 729; + const ENFORCED_SYMBOL = 730; + const ARRAY_SYMBOL = 731; + const OJ_SYMBOL = 732; + const MEMBER_SYMBOL = 733; + const RANDOM_SYMBOL = 734; + const MASTER_COMPRESSION_ALGORITHM_SYMBOL = 735; + const MASTER_ZSTD_COMPRESSION_LEVEL_SYMBOL = 736; + const PRIVILEGE_CHECKS_USER_SYMBOL = 737; + const MASTER_TLS_CIPHERSUITES_SYMBOL = 738; + const REQUIRE_ROW_FORMAT_SYMBOL = 739; + const PASSWORD_LOCK_TIME_SYMBOL = 740; + const FAILED_LOGIN_ATTEMPTS_SYMBOL = 741; + const REQUIRE_TABLE_PRIMARY_KEY_CHECK_SYMBOL = 742; + const STREAM_SYMBOL = 743; + const OFF_SYMBOL = 744; + + /** + * Additional tokens, mostly mirroring the MySQL Workbench lexer grammar. + * + * These tokens are defined in the MySQL Workbench "MySQLLexer.g4" grammar. + * + * See: + * https://github.com/mysql/mysql-workbench/blob/8.0.38/library/parsers/grammars/MySQLLexer.g4 + */ + + // Punctuators + const AT_AT_SIGN_SYMBOL = 745; + const AT_SIGN_SYMBOL = 746; + const CLOSE_CURLY_SYMBOL = 747; + const CLOSE_PAR_SYMBOL = 748; + const COLON_SYMBOL = 749; + const COMMA_SYMBOL = 750; + const DOT_SYMBOL = 751; + const OPEN_CURLY_SYMBOL = 752; + const OPEN_PAR_SYMBOL = 753; + const PARAM_MARKER = 754; + const SEMICOLON_SYMBOL = 755; + + // Operators + const ASSIGN_OPERATOR = 756; + const BITWISE_AND_OPERATOR = 757; + const BITWISE_NOT_OPERATOR = 758; + const BITWISE_OR_OPERATOR = 759; + const BITWISE_XOR_OPERATOR = 760; + const CONCAT_PIPES_SYMBOL = 761; + const DIV_OPERATOR = 762; + const EQUAL_OPERATOR = 763; + const GREATER_OR_EQUAL_OPERATOR = 764; + const GREATER_THAN_OPERATOR = 765; + const JSON_SEPARATOR_SYMBOL = 766; + const JSON_UNQUOTED_SEPARATOR_SYMBOL = 767; + const LESS_OR_EQUAL_OPERATOR = 768; + const LESS_THAN_OPERATOR = 769; + const LOGICAL_AND_OPERATOR = 770; + const LOGICAL_NOT_OPERATOR = 771; + const LOGICAL_OR_OPERATOR = 772; + const MINUS_OPERATOR = 773; + const MOD_OPERATOR = 774; + const MULT_OPERATOR = 775; + const NOT_EQUAL_OPERATOR = 776; + const NULL_SAFE_EQUAL_OPERATOR = 777; + const PLUS_OPERATOR = 778; + const SHIFT_LEFT_OPERATOR = 779; + const SHIFT_RIGHT_OPERATOR = 780; + + // Literals + const BACK_TICK_QUOTED_ID = 781; + const BIN_NUMBER = 782; + const DECIMAL_NUMBER = 783; + const DOUBLE_QUOTED_TEXT = 784; + const FLOAT_NUMBER = 785; + const HEX_NUMBER = 786; + const INT_NUMBER = 787; + const LONG_NUMBER = 788; + const NCHAR_TEXT = 789; + const SINGLE_QUOTED_TEXT = 790; + const ULONGLONG_NUMBER = 791; + + // Identifier-like tokens + const AT_TEXT_SUFFIX = 792; + const IDENTIFIER = 793; + const UNDERSCORE_CHARSET = 794; + + // Other tokens + const INT1_SYMBOL = 795; + const INT2_SYMBOL = 796; + const INT3_SYMBOL = 797; + const INT4_SYMBOL = 798; + const INT8_SYMBOL = 799; + const NOT2_SYMBOL = 800; + const NULL2_SYMBOL = 801; + const SQL_TSI_DAY_SYMBOL = 802; + const SQL_TSI_HOUR_SYMBOL = 803; + const SQL_TSI_MICROSECOND_SYMBOL = 804; + const SQL_TSI_MINUTE_SYMBOL = 805; + const SQL_TSI_MONTH_SYMBOL = 806; + const SQL_TSI_QUARTER_SYMBOL = 807; + const SQL_TSI_SECOND_SYMBOL = 808; + const SQL_TSI_WEEK_SYMBOL = 809; + const SQL_TSI_YEAR_SYMBOL = 810; + + /** + * Other tokens, missing in the MySQL Workbench "MySQLLexer.g4" grammar. + * + * These tokens are missing in the "MySQLLexer.g4" grammar, because the MySQL + * Workbench lexer and parser don't cover 100% of the MySQL syntax. + */ + const INTERSECT_SYMBOL = 811; + const ATTRIBUTE_SYMBOL = 812; + const SOURCE_AUTO_POSITION_SYMBOL = 813; + const SOURCE_BIND_SYMBOL = 814; + const SOURCE_COMPRESSION_ALGORITHM_SYMBOL = 815; + const SOURCE_CONNECT_RETRY_SYMBOL = 816; + const SOURCE_CONNECTION_AUTO_FAILOVER_SYMBOL = 817; + const SOURCE_DELAY_SYMBOL = 818; + const SOURCE_HEARTBEAT_PERIOD_SYMBOL = 819; + const SOURCE_HOST_SYMBOL = 820; + const SOURCE_LOG_FILE_SYMBOL = 821; + const SOURCE_LOG_POS_SYMBOL = 822; + const SOURCE_PASSWORD_SYMBOL = 823; + const SOURCE_PORT_SYMBOL = 824; + const SOURCE_PUBLIC_KEY_PATH_SYMBOL = 825; + const SOURCE_RETRY_COUNT_SYMBOL = 826; + const SOURCE_SSL_SYMBOL = 827; + const SOURCE_SSL_CA_SYMBOL = 828; + const SOURCE_SSL_CAPATH_SYMBOL = 829; + const SOURCE_SSL_CERT_SYMBOL = 830; + const SOURCE_SSL_CIPHER_SYMBOL = 831; + const SOURCE_SSL_CRL_SYMBOL = 832; + const SOURCE_SSL_CRLPATH_SYMBOL = 833; + const SOURCE_SSL_KEY_SYMBOL = 834; + const SOURCE_SSL_VERIFY_SERVER_CERT_SYMBOL = 835; + const SOURCE_TLS_CIPHERSUITES_SYMBOL = 836; + const SOURCE_TLS_VERSION_SYMBOL = 837; + const SOURCE_USER_SYMBOL = 838; + const SOURCE_ZSTD_COMPRESSION_LEVEL_SYMBOL = 839; + const GET_SOURCE_PUBLIC_KEY_SYMBOL = 840; + const GTID_ONLY_SYMBOL = 841; + const ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS_SYMBOL = 842; + const ZONE_SYMBOL = 843; + const INNODB_SYMBOL = 844; // From 5.7.11 defined as is_identifier(..., "INNODB") in "sql_yacc.yy". + const TLS_SYMBOL = 845; // Added in 8.0.21. From 8.0.16 defined as is_identifier(..., "TLS") in "sql_yacc.yy". + const REDO_LOG_SYMBOL = 846; // From 8.0.21 defined as is_identifier(..., "REDO_LOG") in "sql_yacc.yy". + const KEYRING_SYMBOL = 847; + const ENGINE_ATTRIBUTE_SYMBOL = 848; + const SECONDARY_ENGINE_ATTRIBUTE_SYMBOL = 849; + const JSON_VALUE_SYMBOL = 850; + const RETURNING_SYMBOL = 851; + const GEOMCOLLECTION_SYMBOL = 852; + + // Comments + const COMMENT = 900; + const MYSQL_COMMENT_START = 901; + const MYSQL_COMMENT_END = 902; + + // Special tokens + const WHITESPACE = 0; + const EOF = -1; + + /** + * A map of SQL keyword string values to their corresponding token types. + * + * This is used for a fast lookup of MySQL keywords during tokenization. + */ + const TOKENS = array( + // Tokens from MySQL 5.7: + 'ACCESSIBLE' => self::ACCESSIBLE_SYMBOL, + 'ACCOUNT' => self::ACCOUNT_SYMBOL, + 'ACTION' => self::ACTION_SYMBOL, + 'ADD' => self::ADD_SYMBOL, + 'ADDDATE' => self::ADDDATE_SYMBOL, + 'AFTER' => self::AFTER_SYMBOL, + 'AGAINST' => self::AGAINST_SYMBOL, + 'AGGREGATE' => self::AGGREGATE_SYMBOL, + 'ALGORITHM' => self::ALGORITHM_SYMBOL, + 'ALL' => self::ALL_SYMBOL, + 'ALTER' => self::ALTER_SYMBOL, + 'ALWAYS' => self::ALWAYS_SYMBOL, + 'ANALYSE' => self::ANALYSE_SYMBOL, + 'ANALYZE' => self::ANALYZE_SYMBOL, + 'AND' => self::AND_SYMBOL, + 'ANY' => self::ANY_SYMBOL, + 'AS' => self::AS_SYMBOL, + 'ASC' => self::ASC_SYMBOL, + 'ASCII' => self::ASCII_SYMBOL, + 'ASENSITIVE' => self::ASENSITIVE_SYMBOL, + 'AT' => self::AT_SYMBOL, + 'ATTRIBUTE' => self::ATTRIBUTE_SYMBOL, + 'AUTHORS' => self::AUTHORS_SYMBOL, + 'AUTO_INCREMENT' => self::AUTO_INCREMENT_SYMBOL, + 'AUTOEXTEND_SIZE' => self::AUTOEXTEND_SIZE_SYMBOL, + 'AVG' => self::AVG_SYMBOL, + 'AVG_ROW_LENGTH' => self::AVG_ROW_LENGTH_SYMBOL, + 'BACKUP' => self::BACKUP_SYMBOL, + 'BEFORE' => self::BEFORE_SYMBOL, + 'BEGIN' => self::BEGIN_SYMBOL, + 'BETWEEN' => self::BETWEEN_SYMBOL, + 'BIGINT' => self::BIGINT_SYMBOL, + 'BIN_NUM' => self::BIN_NUM_SYMBOL, + 'BINARY' => self::BINARY_SYMBOL, + 'BINLOG' => self::BINLOG_SYMBOL, + 'BIT' => self::BIT_SYMBOL, + 'BIT_AND' => self::BIT_AND_SYMBOL, + 'BIT_OR' => self::BIT_OR_SYMBOL, + 'BIT_XOR' => self::BIT_XOR_SYMBOL, + 'BLOB' => self::BLOB_SYMBOL, + 'BLOCK' => self::BLOCK_SYMBOL, + 'BOOL' => self::BOOL_SYMBOL, + 'BOOLEAN' => self::BOOLEAN_SYMBOL, + 'BOTH' => self::BOTH_SYMBOL, + 'BTREE' => self::BTREE_SYMBOL, + 'BY' => self::BY_SYMBOL, + 'BYTE' => self::BYTE_SYMBOL, + 'CACHE' => self::CACHE_SYMBOL, + 'CALL' => self::CALL_SYMBOL, + 'CASCADE' => self::CASCADE_SYMBOL, + 'CASCADED' => self::CASCADED_SYMBOL, + 'CASE' => self::CASE_SYMBOL, + 'CAST' => self::CAST_SYMBOL, + 'CATALOG_NAME' => self::CATALOG_NAME_SYMBOL, + 'CHAIN' => self::CHAIN_SYMBOL, + 'CHANGE' => self::CHANGE_SYMBOL, + 'CHANGED' => self::CHANGED_SYMBOL, + 'CHANNEL' => self::CHANNEL_SYMBOL, + 'CHAR' => self::CHAR_SYMBOL, + 'CHARACTER' => self::CHARACTER_SYMBOL, + 'CHARSET' => self::CHARSET_SYMBOL, + 'CHECK' => self::CHECK_SYMBOL, + 'CHECKSUM' => self::CHECKSUM_SYMBOL, + 'CIPHER' => self::CIPHER_SYMBOL, + 'CLASS_ORIGIN' => self::CLASS_ORIGIN_SYMBOL, + 'CLIENT' => self::CLIENT_SYMBOL, + 'CLOSE' => self::CLOSE_SYMBOL, + 'COALESCE' => self::COALESCE_SYMBOL, + 'CODE' => self::CODE_SYMBOL, + 'COLLATE' => self::COLLATE_SYMBOL, + 'COLLATION' => self::COLLATION_SYMBOL, + 'COLUMN' => self::COLUMN_SYMBOL, + 'COLUMN_FORMAT' => self::COLUMN_FORMAT_SYMBOL, + 'COLUMN_NAME' => self::COLUMN_NAME_SYMBOL, + 'COLUMNS' => self::COLUMNS_SYMBOL, + 'COMMENT' => self::COMMENT_SYMBOL, + 'COMMIT' => self::COMMIT_SYMBOL, + 'COMMITTED' => self::COMMITTED_SYMBOL, + 'COMPACT' => self::COMPACT_SYMBOL, + 'COMPLETION' => self::COMPLETION_SYMBOL, + 'COMPRESSED' => self::COMPRESSED_SYMBOL, + 'COMPRESSION' => self::COMPRESSION_SYMBOL, + 'CONCURRENT' => self::CONCURRENT_SYMBOL, + 'CONDITION' => self::CONDITION_SYMBOL, + 'CONNECTION' => self::CONNECTION_SYMBOL, + 'CONSISTENT' => self::CONSISTENT_SYMBOL, + 'CONSTRAINT' => self::CONSTRAINT_SYMBOL, + 'CONSTRAINT_CATALOG' => self::CONSTRAINT_CATALOG_SYMBOL, + 'CONSTRAINT_NAME' => self::CONSTRAINT_NAME_SYMBOL, + 'CONSTRAINT_SCHEMA' => self::CONSTRAINT_SCHEMA_SYMBOL, + 'CONTAINS' => self::CONTAINS_SYMBOL, + 'CONTEXT' => self::CONTEXT_SYMBOL, + 'CONTINUE' => self::CONTINUE_SYMBOL, + 'CONTRIBUTORS' => self::CONTRIBUTORS_SYMBOL, + 'CONVERT' => self::CONVERT_SYMBOL, + 'COUNT' => self::COUNT_SYMBOL, + 'CPU' => self::CPU_SYMBOL, + 'CREATE' => self::CREATE_SYMBOL, + 'CROSS' => self::CROSS_SYMBOL, + 'CUBE' => self::CUBE_SYMBOL, + 'CURDATE' => self::CURDATE_SYMBOL, + 'CURRENT' => self::CURRENT_SYMBOL, + 'CURRENT_DATE' => self::CURRENT_DATE_SYMBOL, + 'CURRENT_TIME' => self::CURRENT_TIME_SYMBOL, + 'CURRENT_TIMESTAMP' => self::CURRENT_TIMESTAMP_SYMBOL, + 'CURRENT_USER' => self::CURRENT_USER_SYMBOL, + 'CURSOR' => self::CURSOR_SYMBOL, + 'CURSOR_NAME' => self::CURSOR_NAME_SYMBOL, + 'CURTIME' => self::CURTIME_SYMBOL, + 'DATA' => self::DATA_SYMBOL, + 'DATABASE' => self::DATABASE_SYMBOL, + 'DATABASES' => self::DATABASES_SYMBOL, + 'DATAFILE' => self::DATAFILE_SYMBOL, + 'DATE' => self::DATE_SYMBOL, + 'DATE_ADD' => self::DATE_ADD_SYMBOL, + 'DATE_SUB' => self::DATE_SUB_SYMBOL, + 'DATETIME' => self::DATETIME_SYMBOL, + 'DAY' => self::DAY_SYMBOL, + 'DAY_HOUR' => self::DAY_HOUR_SYMBOL, + 'DAY_MICROSECOND' => self::DAY_MICROSECOND_SYMBOL, + 'DAY_MINUTE' => self::DAY_MINUTE_SYMBOL, + 'DAY_SECOND' => self::DAY_SECOND_SYMBOL, + 'DAYOFMONTH' => self::DAYOFMONTH_SYMBOL, + 'DEALLOCATE' => self::DEALLOCATE_SYMBOL, + 'DEC' => self::DEC_SYMBOL, + 'DECIMAL' => self::DECIMAL_SYMBOL, + 'DECIMAL_NUM' => self::DECIMAL_NUM_SYMBOL, + 'DECLARE' => self::DECLARE_SYMBOL, + 'DEFAULT' => self::DEFAULT_SYMBOL, + 'DEFAULT_AUTH' => self::DEFAULT_AUTH_SYMBOL, + 'DEFINER' => self::DEFINER_SYMBOL, + 'DELAY_KEY_WRITE' => self::DELAY_KEY_WRITE_SYMBOL, + 'DELAYED' => self::DELAYED_SYMBOL, + 'DELETE' => self::DELETE_SYMBOL, + 'DES_KEY_FILE' => self::DES_KEY_FILE_SYMBOL, + 'DESC' => self::DESC_SYMBOL, + 'DESCRIBE' => self::DESCRIBE_SYMBOL, + 'DETERMINISTIC' => self::DETERMINISTIC_SYMBOL, + 'DIAGNOSTICS' => self::DIAGNOSTICS_SYMBOL, + 'DIRECTORY' => self::DIRECTORY_SYMBOL, + 'DISABLE' => self::DISABLE_SYMBOL, + 'DISCARD' => self::DISCARD_SYMBOL, + 'DISK' => self::DISK_SYMBOL, + 'DISTINCT' => self::DISTINCT_SYMBOL, + 'DISTINCTROW' => self::DISTINCTROW_SYMBOL, + 'DIV' => self::DIV_SYMBOL, + 'DO' => self::DO_SYMBOL, + 'DOUBLE' => self::DOUBLE_SYMBOL, + 'DROP' => self::DROP_SYMBOL, + 'DUAL' => self::DUAL_SYMBOL, + 'DUMPFILE' => self::DUMPFILE_SYMBOL, + 'DUPLICATE' => self::DUPLICATE_SYMBOL, + 'DYNAMIC' => self::DYNAMIC_SYMBOL, + 'EACH' => self::EACH_SYMBOL, + 'ELSE' => self::ELSE_SYMBOL, + 'ELSEIF' => self::ELSEIF_SYMBOL, + 'ENABLE' => self::ENABLE_SYMBOL, + 'ENCLOSED' => self::ENCLOSED_SYMBOL, + 'ENCRYPTION' => self::ENCRYPTION_SYMBOL, + 'END' => self::END_SYMBOL, + 'END_OF_INPUT' => self::EOF, + 'ENDS' => self::ENDS_SYMBOL, + 'ENGINE' => self::ENGINE_SYMBOL, + 'ENGINES' => self::ENGINES_SYMBOL, + 'ENUM' => self::ENUM_SYMBOL, + 'ERROR' => self::ERROR_SYMBOL, + 'ERRORS' => self::ERRORS_SYMBOL, + 'ESCAPE' => self::ESCAPE_SYMBOL, + 'ESCAPED' => self::ESCAPED_SYMBOL, + 'EVENT' => self::EVENT_SYMBOL, + 'EVENTS' => self::EVENTS_SYMBOL, + 'EVERY' => self::EVERY_SYMBOL, + 'EXCHANGE' => self::EXCHANGE_SYMBOL, + 'EXECUTE' => self::EXECUTE_SYMBOL, + 'EXISTS' => self::EXISTS_SYMBOL, + 'EXIT' => self::EXIT_SYMBOL, + 'EXPANSION' => self::EXPANSION_SYMBOL, + 'EXPIRE' => self::EXPIRE_SYMBOL, + 'EXPLAIN' => self::EXPLAIN_SYMBOL, + 'EXPORT' => self::EXPORT_SYMBOL, + 'EXTENDED' => self::EXTENDED_SYMBOL, + 'EXTENT_SIZE' => self::EXTENT_SIZE_SYMBOL, + 'EXTRACT' => self::EXTRACT_SYMBOL, + 'FALSE' => self::FALSE_SYMBOL, + 'FAST' => self::FAST_SYMBOL, + 'FAULTS' => self::FAULTS_SYMBOL, + 'FETCH' => self::FETCH_SYMBOL, + 'FIELDS' => self::FIELDS_SYMBOL, + 'FILE' => self::FILE_SYMBOL, + 'FILE_BLOCK_SIZE' => self::FILE_BLOCK_SIZE_SYMBOL, + 'FILTER' => self::FILTER_SYMBOL, + 'FIRST' => self::FIRST_SYMBOL, + 'FIXED' => self::FIXED_SYMBOL, + 'FLOAT' => self::FLOAT_SYMBOL, + 'FLOAT4' => self::FLOAT4_SYMBOL, + 'FLOAT8' => self::FLOAT8_SYMBOL, + 'FLUSH' => self::FLUSH_SYMBOL, + 'FOLLOWS' => self::FOLLOWS_SYMBOL, + 'FOR' => self::FOR_SYMBOL, + 'FORCE' => self::FORCE_SYMBOL, + 'FOREIGN' => self::FOREIGN_SYMBOL, + 'FORMAT' => self::FORMAT_SYMBOL, + 'FOUND' => self::FOUND_SYMBOL, + 'FROM' => self::FROM_SYMBOL, + 'FULL' => self::FULL_SYMBOL, + 'FULLTEXT' => self::FULLTEXT_SYMBOL, + 'FUNCTION' => self::FUNCTION_SYMBOL, + 'GENERAL' => self::GENERAL_SYMBOL, + 'GENERATED' => self::GENERATED_SYMBOL, + 'GEOMCOLLECTION' => self::GEOMCOLLECTION_SYMBOL, + 'GEOMETRY' => self::GEOMETRY_SYMBOL, + 'GEOMETRYCOLLECTION' => self::GEOMETRYCOLLECTION_SYMBOL, + 'GET' => self::GET_SYMBOL, + 'GET_FORMAT' => self::GET_FORMAT_SYMBOL, + 'GLOBAL' => self::GLOBAL_SYMBOL, + 'GRANT' => self::GRANT_SYMBOL, + 'GRANTS' => self::GRANTS_SYMBOL, + 'GROUP' => self::GROUP_SYMBOL, + 'GROUP_CONCAT' => self::GROUP_CONCAT_SYMBOL, + 'GROUP_REPLICATION' => self::GROUP_REPLICATION_SYMBOL, + 'HANDLER' => self::HANDLER_SYMBOL, + 'HASH' => self::HASH_SYMBOL, + 'HAVING' => self::HAVING_SYMBOL, + 'HELP' => self::HELP_SYMBOL, + 'HIGH_PRIORITY' => self::HIGH_PRIORITY_SYMBOL, + 'HOST' => self::HOST_SYMBOL, + 'HOSTS' => self::HOSTS_SYMBOL, + 'HOUR' => self::HOUR_SYMBOL, + 'HOUR_MICROSECOND' => self::HOUR_MICROSECOND_SYMBOL, + 'HOUR_MINUTE' => self::HOUR_MINUTE_SYMBOL, + 'HOUR_SECOND' => self::HOUR_SECOND_SYMBOL, + 'IDENTIFIED' => self::IDENTIFIED_SYMBOL, + 'IF' => self::IF_SYMBOL, + 'IGNORE' => self::IGNORE_SYMBOL, + 'IGNORE_SERVER_IDS' => self::IGNORE_SERVER_IDS_SYMBOL, + 'IMPORT' => self::IMPORT_SYMBOL, + 'IN' => self::IN_SYMBOL, + 'INDEX' => self::INDEX_SYMBOL, + 'INDEXES' => self::INDEXES_SYMBOL, + 'INFILE' => self::INFILE_SYMBOL, + 'INITIAL_SIZE' => self::INITIAL_SIZE_SYMBOL, + 'INNER' => self::INNER_SYMBOL, + 'INNODB' => self::INNODB_SYMBOL, + 'INOUT' => self::INOUT_SYMBOL, + 'INSENSITIVE' => self::INSENSITIVE_SYMBOL, + 'INSERT' => self::INSERT_SYMBOL, + 'INSERT_METHOD' => self::INSERT_METHOD_SYMBOL, + 'INSTALL' => self::INSTALL_SYMBOL, + 'INSTANCE' => self::INSTANCE_SYMBOL, + 'INT' => self::INT_SYMBOL, + 'INT1' => self::INT1_SYMBOL, + 'INT2' => self::INT2_SYMBOL, + 'INT3' => self::INT3_SYMBOL, + 'INT4' => self::INT4_SYMBOL, + 'INT8' => self::INT8_SYMBOL, + 'INTEGER' => self::INTEGER_SYMBOL, + 'INTERVAL' => self::INTERVAL_SYMBOL, + 'INTO' => self::INTO_SYMBOL, + 'INVOKER' => self::INVOKER_SYMBOL, + 'IO' => self::IO_SYMBOL, + 'IO_AFTER_GTIDS' => self::IO_AFTER_GTIDS_SYMBOL, + 'IO_BEFORE_GTIDS' => self::IO_BEFORE_GTIDS_SYMBOL, + 'IO_THREAD' => self::IO_THREAD_SYMBOL, + 'IPC' => self::IPC_SYMBOL, + 'IS' => self::IS_SYMBOL, + 'ISOLATION' => self::ISOLATION_SYMBOL, + 'ISSUER' => self::ISSUER_SYMBOL, + 'ITERATE' => self::ITERATE_SYMBOL, + 'JOIN' => self::JOIN_SYMBOL, + 'JSON' => self::JSON_SYMBOL, + 'KEY' => self::KEY_SYMBOL, + 'KEY_BLOCK_SIZE' => self::KEY_BLOCK_SIZE_SYMBOL, + 'KEYS' => self::KEYS_SYMBOL, + 'KILL' => self::KILL_SYMBOL, + 'LANGUAGE' => self::LANGUAGE_SYMBOL, + 'LAST' => self::LAST_SYMBOL, + 'LEADING' => self::LEADING_SYMBOL, + 'LEAVE' => self::LEAVE_SYMBOL, + 'LEAVES' => self::LEAVES_SYMBOL, + 'LEFT' => self::LEFT_SYMBOL, + 'LESS' => self::LESS_SYMBOL, + 'LEVEL' => self::LEVEL_SYMBOL, + 'LIKE' => self::LIKE_SYMBOL, + 'LIMIT' => self::LIMIT_SYMBOL, + 'LINEAR' => self::LINEAR_SYMBOL, + 'LINES' => self::LINES_SYMBOL, + 'LINESTRING' => self::LINESTRING_SYMBOL, + 'LIST' => self::LIST_SYMBOL, + 'LOAD' => self::LOAD_SYMBOL, + 'LOCAL' => self::LOCAL_SYMBOL, + 'LOCALTIME' => self::LOCALTIME_SYMBOL, + 'LOCALTIMESTAMP' => self::LOCALTIMESTAMP_SYMBOL, + 'LOCATOR' => self::LOCATOR_SYMBOL, + 'LOCK' => self::LOCK_SYMBOL, + 'LOCKS' => self::LOCKS_SYMBOL, + 'LOGFILE' => self::LOGFILE_SYMBOL, + 'LOGS' => self::LOGS_SYMBOL, + 'LONG' => self::LONG_SYMBOL, + 'LONG_NUM' => self::LONG_NUM_SYMBOL, + 'LONGBLOB' => self::LONGBLOB_SYMBOL, + 'LONGTEXT' => self::LONGTEXT_SYMBOL, + 'LOOP' => self::LOOP_SYMBOL, + 'LOW_PRIORITY' => self::LOW_PRIORITY_SYMBOL, + 'MASTER' => self::MASTER_SYMBOL, + 'MASTER_AUTO_POSITION' => self::MASTER_AUTO_POSITION_SYMBOL, + 'MASTER_BIND' => self::MASTER_BIND_SYMBOL, + 'MASTER_CONNECT_RETRY' => self::MASTER_CONNECT_RETRY_SYMBOL, + 'MASTER_DELAY' => self::MASTER_DELAY_SYMBOL, + 'MASTER_HEARTBEAT_PERIOD' => self::MASTER_HEARTBEAT_PERIOD_SYMBOL, + 'MASTER_HOST' => self::MASTER_HOST_SYMBOL, + 'MASTER_LOG_FILE' => self::MASTER_LOG_FILE_SYMBOL, + 'MASTER_LOG_POS' => self::MASTER_LOG_POS_SYMBOL, + 'MASTER_PASSWORD' => self::MASTER_PASSWORD_SYMBOL, + 'MASTER_PORT' => self::MASTER_PORT_SYMBOL, + 'MASTER_RETRY_COUNT' => self::MASTER_RETRY_COUNT_SYMBOL, + 'MASTER_SERVER_ID' => self::MASTER_SERVER_ID_SYMBOL, + 'MASTER_SSL' => self::MASTER_SSL_SYMBOL, + 'MASTER_SSL_CA' => self::MASTER_SSL_CA_SYMBOL, + 'MASTER_SSL_CAPATH' => self::MASTER_SSL_CAPATH_SYMBOL, + 'MASTER_SSL_CERT' => self::MASTER_SSL_CERT_SYMBOL, + 'MASTER_SSL_CIPHER' => self::MASTER_SSL_CIPHER_SYMBOL, + 'MASTER_SSL_CRL' => self::MASTER_SSL_CRL_SYMBOL, + 'MASTER_SSL_CRLPATH' => self::MASTER_SSL_CRLPATH_SYMBOL, + 'MASTER_SSL_KEY' => self::MASTER_SSL_KEY_SYMBOL, + 'MASTER_SSL_VERIFY_SERVER_CERT' => self::MASTER_SSL_VERIFY_SERVER_CERT_SYMBOL, + 'MASTER_TLS_VERSION' => self::MASTER_TLS_VERSION_SYMBOL, + 'MASTER_USER' => self::MASTER_USER_SYMBOL, + 'MATCH' => self::MATCH_SYMBOL, + 'MAX' => self::MAX_SYMBOL, + 'MAX_CONNECTIONS_PER_HOUR' => self::MAX_CONNECTIONS_PER_HOUR_SYMBOL, + 'MAX_QUERIES_PER_HOUR' => self::MAX_QUERIES_PER_HOUR_SYMBOL, + 'MAX_ROWS' => self::MAX_ROWS_SYMBOL, + 'MAX_SIZE' => self::MAX_SIZE_SYMBOL, + 'MAX_STATEMENT_TIME' => self::MAX_STATEMENT_TIME_SYMBOL, + 'MAX_UPDATES_PER_HOUR' => self::MAX_UPDATES_PER_HOUR_SYMBOL, + 'MAX_USER_CONNECTIONS' => self::MAX_USER_CONNECTIONS_SYMBOL, + 'MAXVALUE' => self::MAXVALUE_SYMBOL, + 'MEDIUM' => self::MEDIUM_SYMBOL, + 'MEDIUMBLOB' => self::MEDIUMBLOB_SYMBOL, + 'MEDIUMINT' => self::MEDIUMINT_SYMBOL, + 'MEDIUMTEXT' => self::MEDIUMTEXT_SYMBOL, + 'MEMORY' => self::MEMORY_SYMBOL, + 'MERGE' => self::MERGE_SYMBOL, + 'MESSAGE_TEXT' => self::MESSAGE_TEXT_SYMBOL, + 'MICROSECOND' => self::MICROSECOND_SYMBOL, + 'MID' => self::MID_SYMBOL, + 'MIDDLEINT' => self::MIDDLEINT_SYMBOL, + 'MIGRATE' => self::MIGRATE_SYMBOL, + 'MIN' => self::MIN_SYMBOL, + 'MIN_ROWS' => self::MIN_ROWS_SYMBOL, + 'MINUTE' => self::MINUTE_SYMBOL, + 'MINUTE_MICROSECOND' => self::MINUTE_MICROSECOND_SYMBOL, + 'MINUTE_SECOND' => self::MINUTE_SECOND_SYMBOL, + 'MOD' => self::MOD_SYMBOL, + 'MODE' => self::MODE_SYMBOL, + 'MODIFIES' => self::MODIFIES_SYMBOL, + 'MODIFY' => self::MODIFY_SYMBOL, + 'MONTH' => self::MONTH_SYMBOL, + 'MULTILINESTRING' => self::MULTILINESTRING_SYMBOL, + 'MULTIPOINT' => self::MULTIPOINT_SYMBOL, + 'MULTIPOLYGON' => self::MULTIPOLYGON_SYMBOL, + 'MUTEX' => self::MUTEX_SYMBOL, + 'MYSQL_ERRNO' => self::MYSQL_ERRNO_SYMBOL, + 'NAME' => self::NAME_SYMBOL, + 'NAMES' => self::NAMES_SYMBOL, + 'NATIONAL' => self::NATIONAL_SYMBOL, + 'NATURAL' => self::NATURAL_SYMBOL, + 'NCHAR' => self::NCHAR_SYMBOL, + 'NCHAR_STRING' => self::NCHAR_STRING_SYMBOL, + 'NDB' => self::NDB_SYMBOL, + 'NDBCLUSTER' => self::NDBCLUSTER_SYMBOL, + 'NEG' => self::NEG_SYMBOL, + 'NEVER' => self::NEVER_SYMBOL, + 'NEW' => self::NEW_SYMBOL, + 'NEXT' => self::NEXT_SYMBOL, + 'NO' => self::NO_SYMBOL, + 'NO_WAIT' => self::NO_WAIT_SYMBOL, + 'NO_WRITE_TO_BINLOG' => self::NO_WRITE_TO_BINLOG_SYMBOL, + 'NODEGROUP' => self::NODEGROUP_SYMBOL, + 'NONBLOCKING' => self::NONBLOCKING_SYMBOL, + 'NONE' => self::NONE_SYMBOL, + 'NOT' => self::NOT_SYMBOL, + 'NOW' => self::NOW_SYMBOL, + 'NULL' => self::NULL_SYMBOL, + 'NUMBER' => self::NUMBER_SYMBOL, + 'NUMERIC' => self::NUMERIC_SYMBOL, + 'NVARCHAR' => self::NVARCHAR_SYMBOL, + 'OFFLINE' => self::OFFLINE_SYMBOL, + 'OFFSET' => self::OFFSET_SYMBOL, + 'OLD_PASSWORD' => self::OLD_PASSWORD_SYMBOL, + 'ON' => self::ON_SYMBOL, + 'ONE' => self::ONE_SYMBOL, + 'ONLINE' => self::ONLINE_SYMBOL, + 'ONLY' => self::ONLY_SYMBOL, + 'OPEN' => self::OPEN_SYMBOL, + 'OPTIMIZE' => self::OPTIMIZE_SYMBOL, + 'OPTIMIZER_COSTS' => self::OPTIMIZER_COSTS_SYMBOL, + 'OPTION' => self::OPTION_SYMBOL, + 'OPTIONALLY' => self::OPTIONALLY_SYMBOL, + 'OPTIONS' => self::OPTIONS_SYMBOL, + 'OR' => self::OR_SYMBOL, + 'ORDER' => self::ORDER_SYMBOL, + 'OUT' => self::OUT_SYMBOL, + 'OUTER' => self::OUTER_SYMBOL, + 'OUTFILE' => self::OUTFILE_SYMBOL, + 'OWNER' => self::OWNER_SYMBOL, + 'PACK_KEYS' => self::PACK_KEYS_SYMBOL, + 'PAGE' => self::PAGE_SYMBOL, + 'PARSER' => self::PARSER_SYMBOL, + 'PARTIAL' => self::PARTIAL_SYMBOL, + 'PARTITION' => self::PARTITION_SYMBOL, + 'PARTITIONING' => self::PARTITIONING_SYMBOL, + 'PARTITIONS' => self::PARTITIONS_SYMBOL, + 'PASSWORD' => self::PASSWORD_SYMBOL, + 'PHASE' => self::PHASE_SYMBOL, + 'PLUGIN' => self::PLUGIN_SYMBOL, + 'PLUGIN_DIR' => self::PLUGIN_DIR_SYMBOL, + 'PLUGINS' => self::PLUGINS_SYMBOL, + 'POINT' => self::POINT_SYMBOL, + 'POLYGON' => self::POLYGON_SYMBOL, + 'PORT' => self::PORT_SYMBOL, + 'POSITION' => self::POSITION_SYMBOL, + 'PRECEDES' => self::PRECEDES_SYMBOL, + 'PRECISION' => self::PRECISION_SYMBOL, + 'PREPARE' => self::PREPARE_SYMBOL, + 'PRESERVE' => self::PRESERVE_SYMBOL, + 'PREV' => self::PREV_SYMBOL, + 'PRIMARY' => self::PRIMARY_SYMBOL, + 'PRIVILEGES' => self::PRIVILEGES_SYMBOL, + 'PROCEDURE' => self::PROCEDURE_SYMBOL, + 'PROCESS' => self::PROCESS_SYMBOL, + 'PROCESSLIST' => self::PROCESSLIST_SYMBOL, + 'PROFILE' => self::PROFILE_SYMBOL, + 'PROFILES' => self::PROFILES_SYMBOL, + 'PROXY' => self::PROXY_SYMBOL, + 'PURGE' => self::PURGE_SYMBOL, + 'QUARTER' => self::QUARTER_SYMBOL, + 'QUERY' => self::QUERY_SYMBOL, + 'QUICK' => self::QUICK_SYMBOL, + 'RANGE' => self::RANGE_SYMBOL, + 'READ' => self::READ_SYMBOL, + 'READ_ONLY' => self::READ_ONLY_SYMBOL, + 'READ_WRITE' => self::READ_WRITE_SYMBOL, + 'READS' => self::READS_SYMBOL, + 'REAL' => self::REAL_SYMBOL, + 'REBUILD' => self::REBUILD_SYMBOL, + 'RECOVER' => self::RECOVER_SYMBOL, + 'REDO_BUFFER_SIZE' => self::REDO_BUFFER_SIZE_SYMBOL, + 'REDOFILE' => self::REDOFILE_SYMBOL, + 'REDUNDANT' => self::REDUNDANT_SYMBOL, + 'REFERENCES' => self::REFERENCES_SYMBOL, + 'REGEXP' => self::REGEXP_SYMBOL, + 'RELAY' => self::RELAY_SYMBOL, + 'RELAY_LOG_FILE' => self::RELAY_LOG_FILE_SYMBOL, + 'RELAY_LOG_POS' => self::RELAY_LOG_POS_SYMBOL, + 'RELAY_THREAD' => self::RELAY_THREAD_SYMBOL, + 'RELAYLOG' => self::RELAYLOG_SYMBOL, + 'RELEASE' => self::RELEASE_SYMBOL, + 'RELOAD' => self::RELOAD_SYMBOL, + 'REMOVE' => self::REMOVE_SYMBOL, + 'RENAME' => self::RENAME_SYMBOL, + 'REORGANIZE' => self::REORGANIZE_SYMBOL, + 'REPAIR' => self::REPAIR_SYMBOL, + 'REPEAT' => self::REPEAT_SYMBOL, + 'REPEATABLE' => self::REPEATABLE_SYMBOL, + 'REPLACE' => self::REPLACE_SYMBOL, + 'REPLICATE_DO_DB' => self::REPLICATE_DO_DB_SYMBOL, + 'REPLICATE_DO_TABLE' => self::REPLICATE_DO_TABLE_SYMBOL, + 'REPLICATE_IGNORE_DB' => self::REPLICATE_IGNORE_DB_SYMBOL, + 'REPLICATE_IGNORE_TABLE' => self::REPLICATE_IGNORE_TABLE_SYMBOL, + 'REPLICATE_REWRITE_DB' => self::REPLICATE_REWRITE_DB_SYMBOL, + 'REPLICATE_WILD_DO_TABLE' => self::REPLICATE_WILD_DO_TABLE_SYMBOL, + 'REPLICATE_WILD_IGNORE_TABLE' => self::REPLICATE_WILD_IGNORE_TABLE_SYMBOL, + 'REPLICATION' => self::REPLICATION_SYMBOL, + 'REQUIRE' => self::REQUIRE_SYMBOL, + 'RESET' => self::RESET_SYMBOL, + 'RESIGNAL' => self::RESIGNAL_SYMBOL, + 'RESTORE' => self::RESTORE_SYMBOL, + 'RESTRICT' => self::RESTRICT_SYMBOL, + 'RESUME' => self::RESUME_SYMBOL, + 'RETURN' => self::RETURN_SYMBOL, + 'RETURNED_SQLSTATE' => self::RETURNED_SQLSTATE_SYMBOL, + 'RETURNS' => self::RETURNS_SYMBOL, + 'REVERSE' => self::REVERSE_SYMBOL, + 'REVOKE' => self::REVOKE_SYMBOL, + 'RIGHT' => self::RIGHT_SYMBOL, + 'RLIKE' => self::RLIKE_SYMBOL, + 'ROLLBACK' => self::ROLLBACK_SYMBOL, + 'ROLLUP' => self::ROLLUP_SYMBOL, + 'ROTATE' => self::ROTATE_SYMBOL, + 'ROUTINE' => self::ROUTINE_SYMBOL, + 'ROW' => self::ROW_SYMBOL, + 'ROW_COUNT' => self::ROW_COUNT_SYMBOL, + 'ROW_FORMAT' => self::ROW_FORMAT_SYMBOL, + 'ROWS' => self::ROWS_SYMBOL, + 'RTREE' => self::RTREE_SYMBOL, + 'SAVEPOINT' => self::SAVEPOINT_SYMBOL, + 'SCHEDULE' => self::SCHEDULE_SYMBOL, + 'SCHEMA' => self::SCHEMA_SYMBOL, + 'SCHEMA_NAME' => self::SCHEMA_NAME_SYMBOL, + 'SCHEMAS' => self::SCHEMAS_SYMBOL, + 'SECOND' => self::SECOND_SYMBOL, + 'SECOND_MICROSECOND' => self::SECOND_MICROSECOND_SYMBOL, + 'SECURITY' => self::SECURITY_SYMBOL, + 'SELECT' => self::SELECT_SYMBOL, + 'SENSITIVE' => self::SENSITIVE_SYMBOL, + 'SEPARATOR' => self::SEPARATOR_SYMBOL, + 'SERIAL' => self::SERIAL_SYMBOL, + 'SERIALIZABLE' => self::SERIALIZABLE_SYMBOL, + 'SERVER' => self::SERVER_SYMBOL, + 'SERVER_OPTIONS' => self::SERVER_OPTIONS_SYMBOL, + 'SESSION' => self::SESSION_SYMBOL, + 'SESSION_USER' => self::SESSION_USER_SYMBOL, + 'SET' => self::SET_SYMBOL, + 'SET_VAR' => self::SET_VAR_SYMBOL, + 'SHARE' => self::SHARE_SYMBOL, + 'SHOW' => self::SHOW_SYMBOL, + 'SHUTDOWN' => self::SHUTDOWN_SYMBOL, + 'SIGNAL' => self::SIGNAL_SYMBOL, + 'SIGNED' => self::SIGNED_SYMBOL, + 'SIMPLE' => self::SIMPLE_SYMBOL, + 'SLAVE' => self::SLAVE_SYMBOL, + 'SLOW' => self::SLOW_SYMBOL, + 'SMALLINT' => self::SMALLINT_SYMBOL, + 'SNAPSHOT' => self::SNAPSHOT_SYMBOL, + 'SOCKET' => self::SOCKET_SYMBOL, + 'SOME' => self::SOME_SYMBOL, + 'SONAME' => self::SONAME_SYMBOL, + 'SOUNDS' => self::SOUNDS_SYMBOL, + 'SOURCE' => self::SOURCE_SYMBOL, + 'SPATIAL' => self::SPATIAL_SYMBOL, + 'SPECIFIC' => self::SPECIFIC_SYMBOL, + 'SQL' => self::SQL_SYMBOL, + 'SQL_AFTER_GTIDS' => self::SQL_AFTER_GTIDS_SYMBOL, + 'SQL_AFTER_MTS_GAPS' => self::SQL_AFTER_MTS_GAPS_SYMBOL, + 'SQL_BEFORE_GTIDS' => self::SQL_BEFORE_GTIDS_SYMBOL, + 'SQL_BIG_RESULT' => self::SQL_BIG_RESULT_SYMBOL, + 'SQL_BUFFER_RESULT' => self::SQL_BUFFER_RESULT_SYMBOL, + 'SQL_CACHE' => self::SQL_CACHE_SYMBOL, + 'SQL_CALC_FOUND_ROWS' => self::SQL_CALC_FOUND_ROWS_SYMBOL, + 'SQL_NO_CACHE' => self::SQL_NO_CACHE_SYMBOL, + 'SQL_SMALL_RESULT' => self::SQL_SMALL_RESULT_SYMBOL, + 'SQL_THREAD' => self::SQL_THREAD_SYMBOL, + 'SQL_TSI_DAY' => self::SQL_TSI_DAY_SYMBOL, + 'SQL_TSI_HOUR' => self::SQL_TSI_HOUR_SYMBOL, + 'SQL_TSI_MICROSECOND' => self::SQL_TSI_MICROSECOND_SYMBOL, + 'SQL_TSI_MINUTE' => self::SQL_TSI_MINUTE_SYMBOL, + 'SQL_TSI_MONTH' => self::SQL_TSI_MONTH_SYMBOL, + 'SQL_TSI_QUARTER' => self::SQL_TSI_QUARTER_SYMBOL, + 'SQL_TSI_SECOND' => self::SQL_TSI_SECOND_SYMBOL, + 'SQL_TSI_WEEK' => self::SQL_TSI_WEEK_SYMBOL, + 'SQL_TSI_YEAR' => self::SQL_TSI_YEAR_SYMBOL, + 'SQLEXCEPTION' => self::SQLEXCEPTION_SYMBOL, + 'SQLSTATE' => self::SQLSTATE_SYMBOL, + 'SQLWARNING' => self::SQLWARNING_SYMBOL, + 'SSL' => self::SSL_SYMBOL, + 'STACKED' => self::STACKED_SYMBOL, + 'START' => self::START_SYMBOL, + 'STARTING' => self::STARTING_SYMBOL, + 'STARTS' => self::STARTS_SYMBOL, + 'STATS_AUTO_RECALC' => self::STATS_AUTO_RECALC_SYMBOL, + 'STATS_PERSISTENT' => self::STATS_PERSISTENT_SYMBOL, + 'STATS_SAMPLE_PAGES' => self::STATS_SAMPLE_PAGES_SYMBOL, + 'STATUS' => self::STATUS_SYMBOL, + 'STD' => self::STD_SYMBOL, + 'STDDEV' => self::STDDEV_SYMBOL, + 'STDDEV_POP' => self::STDDEV_POP_SYMBOL, + 'STDDEV_SAMP' => self::STDDEV_SAMP_SYMBOL, + 'STOP' => self::STOP_SYMBOL, + 'STORAGE' => self::STORAGE_SYMBOL, + 'STORED' => self::STORED_SYMBOL, + 'STRAIGHT_JOIN' => self::STRAIGHT_JOIN_SYMBOL, + 'STRING' => self::STRING_SYMBOL, + 'SUBCLASS_ORIGIN' => self::SUBCLASS_ORIGIN_SYMBOL, + 'SUBDATE' => self::SUBDATE_SYMBOL, + 'SUBJECT' => self::SUBJECT_SYMBOL, + 'SUBPARTITION' => self::SUBPARTITION_SYMBOL, + 'SUBPARTITIONS' => self::SUBPARTITIONS_SYMBOL, + 'SUBSTR' => self::SUBSTR_SYMBOL, + 'SUBSTRING' => self::SUBSTRING_SYMBOL, + 'SUM' => self::SUM_SYMBOL, + 'SUPER' => self::SUPER_SYMBOL, + 'SUSPEND' => self::SUSPEND_SYMBOL, + 'SWAPS' => self::SWAPS_SYMBOL, + 'SWITCHES' => self::SWITCHES_SYMBOL, + 'SYSDATE' => self::SYSDATE_SYMBOL, + 'SYSTEM_USER' => self::SYSTEM_USER_SYMBOL, + 'TABLE' => self::TABLE_SYMBOL, + 'TABLE_CHECKSUM' => self::TABLE_CHECKSUM_SYMBOL, + 'TABLE_NAME' => self::TABLE_NAME_SYMBOL, + 'TABLE_REF_PRIORITY' => self::TABLE_REF_PRIORITY_SYMBOL, + 'TABLES' => self::TABLES_SYMBOL, + 'TABLESPACE' => self::TABLESPACE_SYMBOL, + 'TEMPORARY' => self::TEMPORARY_SYMBOL, + 'TEMPTABLE' => self::TEMPTABLE_SYMBOL, + 'TERMINATED' => self::TERMINATED_SYMBOL, + 'TEXT' => self::TEXT_SYMBOL, + 'THAN' => self::THAN_SYMBOL, + 'THEN' => self::THEN_SYMBOL, + 'TIME' => self::TIME_SYMBOL, + 'TIMESTAMP' => self::TIMESTAMP_SYMBOL, + 'TIMESTAMP_ADD' => self::TIMESTAMP_ADD_SYMBOL, + 'TIMESTAMP_DIFF' => self::TIMESTAMP_DIFF_SYMBOL, + 'TINYBLOB' => self::TINYBLOB_SYMBOL, + 'TINYINT' => self::TINYINT_SYMBOL, + 'TINYTEXT' => self::TINYTEXT_SYMBOL, + 'TO' => self::TO_SYMBOL, + 'TRAILING' => self::TRAILING_SYMBOL, + 'TRANSACTION' => self::TRANSACTION_SYMBOL, + 'TRIGGER' => self::TRIGGER_SYMBOL, + 'TRIGGERS' => self::TRIGGERS_SYMBOL, + 'TRIM' => self::TRIM_SYMBOL, + 'TRUE' => self::TRUE_SYMBOL, + 'TRUNCATE' => self::TRUNCATE_SYMBOL, + 'TYPE' => self::TYPE_SYMBOL, + 'TYPES' => self::TYPES_SYMBOL, + 'UDF_RETURNS' => self::UDF_RETURNS_SYMBOL, + 'UNCOMMITTED' => self::UNCOMMITTED_SYMBOL, + 'UNDEFINED' => self::UNDEFINED_SYMBOL, + 'UNDO' => self::UNDO_SYMBOL, + 'UNDO_BUFFER_SIZE' => self::UNDO_BUFFER_SIZE_SYMBOL, + 'UNDOFILE' => self::UNDOFILE_SYMBOL, + 'UNICODE' => self::UNICODE_SYMBOL, + 'UNINSTALL' => self::UNINSTALL_SYMBOL, + 'UNION' => self::UNION_SYMBOL, + 'UNIQUE' => self::UNIQUE_SYMBOL, + 'UNKNOWN' => self::UNKNOWN_SYMBOL, + 'UNLOCK' => self::UNLOCK_SYMBOL, + 'UNSIGNED' => self::UNSIGNED_SYMBOL, + 'UNTIL' => self::UNTIL_SYMBOL, + 'UPDATE' => self::UPDATE_SYMBOL, + 'UPGRADE' => self::UPGRADE_SYMBOL, + 'USAGE' => self::USAGE_SYMBOL, + 'USE' => self::USE_SYMBOL, + 'USE_FRM' => self::USE_FRM_SYMBOL, + 'USER' => self::USER_SYMBOL, + 'USER_RESOURCES' => self::USER_RESOURCES_SYMBOL, + 'USING' => self::USING_SYMBOL, + 'UTC_DATE' => self::UTC_DATE_SYMBOL, + 'UTC_TIME' => self::UTC_TIME_SYMBOL, + 'UTC_TIMESTAMP' => self::UTC_TIMESTAMP_SYMBOL, + 'VALIDATION' => self::VALIDATION_SYMBOL, + 'VALUE' => self::VALUE_SYMBOL, + 'VALUES' => self::VALUES_SYMBOL, + 'VAR_POP' => self::VAR_POP_SYMBOL, + 'VAR_SAMP' => self::VAR_SAMP_SYMBOL, + 'VARBINARY' => self::VARBINARY_SYMBOL, + 'VARCHAR' => self::VARCHAR_SYMBOL, + 'VARCHARACTER' => self::VARCHARACTER_SYMBOL, + 'VARIABLES' => self::VARIABLES_SYMBOL, + 'VARIANCE' => self::VARIANCE_SYMBOL, + 'VARYING' => self::VARYING_SYMBOL, + 'VIEW' => self::VIEW_SYMBOL, + 'VIRTUAL' => self::VIRTUAL_SYMBOL, + 'WAIT' => self::WAIT_SYMBOL, + 'WARNINGS' => self::WARNINGS_SYMBOL, + 'WEEK' => self::WEEK_SYMBOL, + 'WEIGHT_STRING' => self::WEIGHT_STRING_SYMBOL, + 'WHEN' => self::WHEN_SYMBOL, + 'WHERE' => self::WHERE_SYMBOL, + 'WHILE' => self::WHILE_SYMBOL, + 'WITH' => self::WITH_SYMBOL, + 'WITHOUT' => self::WITHOUT_SYMBOL, + 'WORK' => self::WORK_SYMBOL, + 'WRAPPER' => self::WRAPPER_SYMBOL, + 'WRITE' => self::WRITE_SYMBOL, + 'X509' => self::X509_SYMBOL, + 'XA' => self::XA_SYMBOL, + 'XID' => self::XID_SYMBOL, + 'XML' => self::XML_SYMBOL, + 'XOR' => self::XOR_SYMBOL, + 'YEAR' => self::YEAR_SYMBOL, + 'YEAR_MONTH' => self::YEAR_MONTH_SYMBOL, + 'ZEROFILL' => self::ZEROFILL_SYMBOL, + + // Tokens from MySQL 8.0: + 'ACTIVE' => self::ACTIVE_SYMBOL, + 'ADMIN' => self::ADMIN_SYMBOL, + 'ARRAY' => self::ARRAY_SYMBOL, + 'ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS' => self::ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS_SYMBOL, + 'BUCKETS' => self::BUCKETS_SYMBOL, + 'CLONE' => self::CLONE_SYMBOL, + 'COMPONENT' => self::COMPONENT_SYMBOL, + 'CUME_DIST' => self::CUME_DIST_SYMBOL, + 'DEFINITION' => self::DEFINITION_SYMBOL, + 'DENSE_RANK' => self::DENSE_RANK_SYMBOL, + 'DESCRIPTION' => self::DESCRIPTION_SYMBOL, + 'EMPTY' => self::EMPTY_SYMBOL, + 'ENFORCED' => self::ENFORCED_SYMBOL, + 'ENGINE_ATTRIBUTE' => self::ENGINE_ATTRIBUTE_SYMBOL, + 'EXCEPT' => self::EXCEPT_SYMBOL, + 'EXCLUDE' => self::EXCLUDE_SYMBOL, + 'FAILED_LOGIN_ATTEMPTS' => self::FAILED_LOGIN_ATTEMPTS_SYMBOL, + 'FIRST_VALUE' => self::FIRST_VALUE_SYMBOL, + 'FOLLOWING' => self::FOLLOWING_SYMBOL, + 'GET_MASTER_PUBLIC_KEY_SYM' => self::GET_MASTER_PUBLIC_KEY_SYMBOL, + 'GET_SOURCE_PUBLIC_KEY' => self::GET_SOURCE_PUBLIC_KEY_SYMBOL, + 'GROUPING' => self::GROUPING_SYMBOL, + 'GROUPS' => self::GROUPS_SYMBOL, + 'GTID_ONLY' => self::GTID_ONLY_SYMBOL, + 'HISTOGRAM' => self::HISTOGRAM_SYMBOL, + 'HISTORY' => self::HISTORY_SYMBOL, + 'INACTIVE' => self::INACTIVE_SYMBOL, + 'INTERSECT' => self::INTERSECT_SYMBOL, + 'INVISIBLE' => self::INVISIBLE_SYMBOL, + 'JSON_ARRAYAGG' => self::JSON_ARRAYAGG_SYMBOL, + 'JSON_OBJECTAGG' => self::JSON_OBJECTAGG_SYMBOL, + 'JSON_TABLE' => self::JSON_TABLE_SYMBOL, + 'JSON_VALUE' => self::JSON_VALUE_SYMBOL, + 'KEYRING' => self::KEYRING_SYMBOL, + 'LAG' => self::LAG_SYMBOL, + 'LAST_VALUE' => self::LAST_VALUE_SYMBOL, + 'LATERAL' => self::LATERAL_SYMBOL, + 'LEAD' => self::LEAD_SYMBOL, + 'LOCKED' => self::LOCKED_SYMBOL, + 'MASTER_COMPRESSION_ALGORITHM' => self::MASTER_COMPRESSION_ALGORITHM_SYMBOL, + 'MASTER_PUBLIC_KEY_PATH' => self::MASTER_PUBLIC_KEY_PATH_SYMBOL, + 'MASTER_TLS_CIPHERSUITES' => self::MASTER_TLS_CIPHERSUITES_SYMBOL, + 'MASTER_ZSTD_COMPRESSION_LEVEL' => self::MASTER_ZSTD_COMPRESSION_LEVEL_SYMBOL, + 'MEMBER' => self::MEMBER_SYMBOL, + 'NESTED' => self::NESTED_SYMBOL, + 'NETWORK_NAMESPACE' => self::NETWORK_NAMESPACE_SYMBOL, + 'NOWAIT' => self::NOWAIT_SYMBOL, + 'NTH_VALUE' => self::NTH_VALUE_SYMBOL, + 'NTILE' => self::NTILE_SYMBOL, + 'NULLS' => self::NULLS_SYMBOL, + 'OF' => self::OF_SYMBOL, + 'OFF' => self::OFF_SYMBOL, + 'OJ' => self::OJ_SYMBOL, + 'OLD' => self::OLD_SYMBOL, + 'OPTIONAL' => self::OPTIONAL_SYMBOL, + 'ORDINALITY' => self::ORDINALITY_SYMBOL, + 'ORGANIZATION' => self::ORGANIZATION_SYMBOL, + 'OTHERS' => self::OTHERS_SYMBOL, + 'OVER' => self::OVER_SYMBOL, + 'PASSWORD_LOCK_TIME' => self::PASSWORD_LOCK_TIME_SYMBOL, + 'PATH' => self::PATH_SYMBOL, + 'PERCENT_RANK' => self::PERCENT_RANK_SYMBOL, + 'PERSIST' => self::PERSIST_SYMBOL, + 'PERSIST_ONLY' => self::PERSIST_ONLY_SYMBOL, + 'PRECEDING' => self::PRECEDING_SYMBOL, + 'PRIVILEGE_CHECKS_USER' => self::PRIVILEGE_CHECKS_USER_SYMBOL, + 'RANDOM' => self::RANDOM_SYMBOL, + 'RANK' => self::RANK_SYMBOL, + 'RECURSIVE' => self::RECURSIVE_SYMBOL, + 'REDO_LOG' => self::REDO_LOG_SYMBOL, + 'REFERENCE' => self::REFERENCE_SYMBOL, + 'REMOTE' => self::REMOTE_SYMBOL, + 'REQUIRE_ROW_FORMAT' => self::REQUIRE_ROW_FORMAT_SYMBOL, + 'REQUIRE_TABLE_PRIMARY_KEY_CHECK' => self::REQUIRE_TABLE_PRIMARY_KEY_CHECK_SYMBOL, + 'RESOURCE' => self::RESOURCE_SYMBOL, + 'RESPECT' => self::RESPECT_SYMBOL, + 'RESTART' => self::RESTART_SYMBOL, + 'RETAIN' => self::RETAIN_SYMBOL, + 'RETURNING' => self::RETURNING_SYMBOL, + 'REUSE' => self::REUSE_SYMBOL, + 'ROLE' => self::ROLE_SYMBOL, + 'ROW_NUMBER' => self::ROW_NUMBER_SYMBOL, + 'SECONDARY' => self::SECONDARY_SYMBOL, + 'SECONDARY_ENGINE' => self::SECONDARY_ENGINE_SYMBOL, + 'SECONDARY_ENGINE_ATTRIBUTE' => self::SECONDARY_ENGINE_ATTRIBUTE_SYMBOL, + 'SECONDARY_LOAD' => self::SECONDARY_LOAD_SYMBOL, + 'SECONDARY_UNLOAD' => self::SECONDARY_UNLOAD_SYMBOL, + 'SKIP' => self::SKIP_SYMBOL, + 'SOURCE_AUTO_POSITION' => self::SOURCE_AUTO_POSITION_SYMBOL, + 'SOURCE_BIND' => self::SOURCE_BIND_SYMBOL, + 'SOURCE_COMPRESSION_ALGORITHM' => self::SOURCE_COMPRESSION_ALGORITHM_SYMBOL, + 'SOURCE_CONNECT_RETRY' => self::SOURCE_CONNECT_RETRY_SYMBOL, + 'SOURCE_CONNECTION_AUTO_FAILOVER' => self::SOURCE_CONNECTION_AUTO_FAILOVER_SYMBOL, + 'SOURCE_DELAY' => self::SOURCE_DELAY_SYMBOL, + 'SOURCE_HEARTBEAT_PERIOD' => self::SOURCE_HEARTBEAT_PERIOD_SYMBOL, + 'SOURCE_HOST' => self::SOURCE_HOST_SYMBOL, + 'SOURCE_LOG_FILE' => self::SOURCE_LOG_FILE_SYMBOL, + 'SOURCE_LOG_POS' => self::SOURCE_LOG_POS_SYMBOL, + 'SOURCE_PASSWORD' => self::SOURCE_PASSWORD_SYMBOL, + 'SOURCE_PORT' => self::SOURCE_PORT_SYMBOL, + 'SOURCE_PUBLIC_KEY_PATH' => self::SOURCE_PUBLIC_KEY_PATH_SYMBOL, + 'SOURCE_RETRY_COUNT' => self::SOURCE_RETRY_COUNT_SYMBOL, + 'SOURCE_SSL' => self::SOURCE_SSL_SYMBOL, + 'SOURCE_SSL_CA' => self::SOURCE_SSL_CA_SYMBOL, + 'SOURCE_SSL_CAPATH' => self::SOURCE_SSL_CAPATH_SYMBOL, + 'SOURCE_SSL_CERT' => self::SOURCE_SSL_CERT_SYMBOL, + 'SOURCE_SSL_CIPHER' => self::SOURCE_SSL_CIPHER_SYMBOL, + 'SOURCE_SSL_CRL' => self::SOURCE_SSL_CRL_SYMBOL, + 'SOURCE_SSL_CRLPATH' => self::SOURCE_SSL_CRLPATH_SYMBOL, + 'SOURCE_SSL_KEY' => self::SOURCE_SSL_KEY_SYMBOL, + 'SOURCE_SSL_VERIFY_SERVER_CERT' => self::SOURCE_SSL_VERIFY_SERVER_CERT_SYMBOL, + 'SOURCE_TLS_CIPHERSUITES' => self::SOURCE_TLS_CIPHERSUITES_SYMBOL, + 'SOURCE_TLS_VERSION' => self::SOURCE_TLS_VERSION_SYMBOL, + 'SOURCE_USER' => self::SOURCE_USER_SYMBOL, + 'SOURCE_ZSTD_COMPRESSION_LEVEL' => self::SOURCE_ZSTD_COMPRESSION_LEVEL_SYMBOL, + 'SRID' => self::SRID_SYMBOL, + 'STREAM' => self::STREAM_SYMBOL, + 'SYSTEM' => self::SYSTEM_SYMBOL, + 'THREAD_PRIORITY' => self::THREAD_PRIORITY_SYMBOL, + 'TIES' => self::TIES_SYMBOL, + 'TLS' => self::TLS_SYMBOL, + 'UNBOUNDED' => self::UNBOUNDED_SYMBOL, + 'VCPU' => self::VCPU_SYMBOL, + 'VISIBLE' => self::VISIBLE_SYMBOL, + 'WINDOW' => self::WINDOW_SYMBOL, + 'ZONE' => self::ZONE_SYMBOL, + ); + + /** + * Tokens that represent function calls when followed by a parenthesis. + */ + const FUNCTIONS = array( + self::ADDDATE_SYMBOL => true, + self::BIT_AND_SYMBOL => true, + self::BIT_OR_SYMBOL => true, + self::BIT_XOR_SYMBOL => true, + self::CAST_SYMBOL => true, + self::COUNT_SYMBOL => true, + self::CURDATE_SYMBOL => true, + self::CURRENT_DATE_SYMBOL => true, + self::CURRENT_TIME_SYMBOL => true, + self::CURTIME_SYMBOL => true, + self::DATE_ADD_SYMBOL => true, + self::DATE_SUB_SYMBOL => true, + self::EXTRACT_SYMBOL => true, + self::GROUP_CONCAT_SYMBOL => true, + self::MAX_SYMBOL => true, + self::MID_SYMBOL => true, + self::MIN_SYMBOL => true, + self::NOW_SYMBOL => true, + self::POSITION_SYMBOL => true, + self::SESSION_USER_SYMBOL => true, + self::STD_SYMBOL => true, + self::STDDEV_POP_SYMBOL => true, + self::STDDEV_SAMP_SYMBOL => true, + self::STDDEV_SYMBOL => true, + self::SUBDATE_SYMBOL => true, + self::SUBSTR_SYMBOL => true, + self::SUBSTRING_SYMBOL => true, + self::SUM_SYMBOL => true, + self::SYSDATE_SYMBOL => true, + self::SYSTEM_USER_SYMBOL => true, + self::TRIM_SYMBOL => true, + self::VAR_POP_SYMBOL => true, + self::VAR_SAMP_SYMBOL => true, + self::VARIANCE_SYMBOL => true, + ); + + /** + * Tokens that are functionally equivalent and can be used interchangeably. + * + * Some of the synonyms may have a different keyword or function status and + * version constraints, hence the synonym conversion needs to be applied + * at the end of the tokenization process, after all other transformations. + * + * E.g.: NOW is a non-reserved keyword that needs to be used with "()" while + * CURRENT_TIMESTAMP is a reserved keyword that can be used without "()". + */ + const SYNONYMS = array( + self::CHARACTER_SYMBOL => self::CHAR_SYMBOL, + self::CURRENT_DATE_SYMBOL => self::CURDATE_SYMBOL, + self::CURRENT_TIME_SYMBOL => self::CURTIME_SYMBOL, + self::CURRENT_TIMESTAMP_SYMBOL => self::NOW_SYMBOL, + self::DAYOFMONTH_SYMBOL => self::DAY_SYMBOL, + self::DEC_SYMBOL => self::DECIMAL_SYMBOL, + self::DISTINCTROW_SYMBOL => self::DISTINCT_SYMBOL, + self::FIELDS_SYMBOL => self::COLUMNS_SYMBOL, + self::FLOAT4_SYMBOL => self::FLOAT_SYMBOL, + self::FLOAT8_SYMBOL => self::DOUBLE_SYMBOL, + self::GEOMCOLLECTION_SYMBOL => self::GEOMETRYCOLLECTION_SYMBOL, + self::INT1_SYMBOL => self::TINYINT_SYMBOL, + self::INT2_SYMBOL => self::SMALLINT_SYMBOL, + self::INT3_SYMBOL => self::MEDIUMINT_SYMBOL, + self::INT4_SYMBOL => self::INT_SYMBOL, + self::INT8_SYMBOL => self::BIGINT_SYMBOL, + self::INTEGER_SYMBOL => self::INT_SYMBOL, + self::IO_THREAD_SYMBOL => self::RELAY_THREAD_SYMBOL, + self::LOCALTIME_SYMBOL => self::NOW_SYMBOL, + self::LOCALTIMESTAMP_SYMBOL => self::NOW_SYMBOL, + self::MID_SYMBOL => self::SUBSTRING_SYMBOL, + self::MIDDLEINT_SYMBOL => self::MEDIUMINT_SYMBOL, + self::NDB_SYMBOL => self::NDBCLUSTER_SYMBOL, + self::RLIKE_SYMBOL => self::REGEXP_SYMBOL, + self::SCHEMA_SYMBOL => self::DATABASE_SYMBOL, + self::SCHEMAS_SYMBOL => self::DATABASES_SYMBOL, + self::SESSION_USER_SYMBOL => self::USER_SYMBOL, + self::SOME_SYMBOL => self::ANY_SYMBOL, + self::SQL_TSI_DAY_SYMBOL => self::DAY_SYMBOL, + self::SQL_TSI_HOUR_SYMBOL => self::HOUR_SYMBOL, + self::SQL_TSI_MICROSECOND_SYMBOL => self::MICROSECOND_SYMBOL, + self::SQL_TSI_MINUTE_SYMBOL => self::MINUTE_SYMBOL, + self::SQL_TSI_MONTH_SYMBOL => self::MONTH_SYMBOL, + self::SQL_TSI_QUARTER_SYMBOL => self::QUARTER_SYMBOL, + self::SQL_TSI_SECOND_SYMBOL => self::SECOND_SYMBOL, + self::SQL_TSI_WEEK_SYMBOL => self::WEEK_SYMBOL, + self::SQL_TSI_YEAR_SYMBOL => self::YEAR_SYMBOL, + self::STDDEV_POP_SYMBOL => self::STD_SYMBOL, + self::STDDEV_SYMBOL => self::STD_SYMBOL, + self::SUBSTR_SYMBOL => self::SUBSTRING_SYMBOL, + self::SYSTEM_USER_SYMBOL => self::USER_SYMBOL, + self::VAR_POP_SYMBOL => self::VARIANCE_SYMBOL, + self::VARCHARACTER_SYMBOL => self::VARCHAR_SYMBOL, + ); + + /** + * Version constraints for version-specific tokens. + * + * This is a map of tokens to the MySQL server versions in which they were + * introduced (positive number) or removed (negative number). Tokens that + * were both introduced and later removed are not included in this list + * and are handled by manual version checks in the tokenization process. + * + * See: + * https://dev.mysql.com/doc/mysqld-version-reference/en/keywords.html + * + * @TODO Verify the version specifiers and ranges against the list above. + * + * Positive number: >= (introduced in ) + * Negative number: < (removed in ) + */ + const VERSIONS = array( + // MySQL 5 + self::ACCOUNT_SYMBOL => 50707, + self::ALWAYS_SYMBOL => 50707, + self::ANALYSE_SYMBOL => -80000, + self::AUTHORS_SYMBOL => -50700, + self::CHANNEL_SYMBOL => 50706, + self::COMPRESSION_SYMBOL => 50707, + self::CONTRIBUTORS_SYMBOL => -50700, + self::CURRENT_SYMBOL => 50604, + self::DEFAULT_AUTH_SYMBOL => 50604, + self::DES_KEY_FILE_SYMBOL => -80003, + self::ENCRYPTION_SYMBOL => 50711, + self::EXPIRE_SYMBOL => 50606, + self::EXPORT_SYMBOL => 50606, + self::FILE_BLOCK_SIZE_SYMBOL => 50707, + self::FILTER_SYMBOL => 50700, + self::FOLLOWS_SYMBOL => 50700, + self::GENERATED_SYMBOL => 50707, + self::GET_SYMBOL => 50604, + self::GROUP_REPLICATION_SYMBOL => 50707, + self::INNODB_SYMBOL => 50711, + self::INSTANCE_SYMBOL => 50713, + self::JSON_SYMBOL => 50708, + self::MASTER_AUTO_POSITION_SYMBOL => 50605, + self::MASTER_BIND_SYMBOL => 50602, + self::MASTER_RETRY_COUNT_SYMBOL => 50601, + self::MASTER_SSL_CRL_SYMBOL => 50603, + self::MASTER_SSL_CRLPATH_SYMBOL => 50603, + self::MASTER_TLS_VERSION_SYMBOL => 50713, + self::NEVER_SYMBOL => 50704, + self::NUMBER_SYMBOL => 50606, + self::OLD_PASSWORD_SYMBOL => -50706, + self::ONLY_SYMBOL => 50605, + self::OPTIMIZER_COSTS_SYMBOL => 50706, + self::PLUGIN_DIR_SYMBOL => 50604, + self::PRECEDES_SYMBOL => 50700, + self::REDOFILE_SYMBOL => -80000, + self::REPLICATE_DO_DB_SYMBOL => 50700, + self::REPLICATE_DO_TABLE_SYMBOL => 50700, + self::REPLICATE_IGNORE_DB_SYMBOL => 50700, + self::REPLICATE_IGNORE_TABLE_SYMBOL => 50700, + self::REPLICATE_REWRITE_DB_SYMBOL => 50700, + self::REPLICATE_WILD_DO_TABLE_SYMBOL => 50700, + self::REPLICATE_WILD_IGNORE_TABLE_SYMBOL => 50700, + self::ROTATE_SYMBOL => 50713, + self::SQL_AFTER_MTS_GAPS_SYMBOL => 50606, + self::SQL_CACHE_SYMBOL => -80000, + self::STACKED_SYMBOL => 50700, + self::STORED_SYMBOL => 50707, + self::TABLE_REF_PRIORITY_SYMBOL => -80000, + self::VALIDATION_SYMBOL => 50706, + self::VIRTUAL_SYMBOL => 50707, + self::XID_SYMBOL => 50704, + + // MySQL 8 + self::ACTIVE_SYMBOL => 80014, + self::ADMIN_SYMBOL => 80000, + self::ARRAY_SYMBOL => 80017, + self::ASSIGN_GTIDS_TO_ANONYMOUS_TRANSACTIONS_SYMBOL => 80000, + self::ATTRIBUTE_SYMBOL => 80021, + self::BUCKETS_SYMBOL => 80000, + self::CLONE_SYMBOL => 80000, + self::COMPONENT_SYMBOL => 80000, + self::CUME_DIST_SYMBOL => 80000, + self::DEFINITION_SYMBOL => 80011, + self::DENSE_RANK_SYMBOL => 80000, + self::DESCRIPTION_SYMBOL => 80011, + self::EMPTY_SYMBOL => 80000, + self::ENFORCED_SYMBOL => 80017, + self::ENGINE_ATTRIBUTE_SYMBOL => 80021, + self::EXCEPT_SYMBOL => 80000, + self::EXCLUDE_SYMBOL => 80000, + self::FAILED_LOGIN_ATTEMPTS_SYMBOL => 80019, + self::FIRST_VALUE_SYMBOL => 80000, + self::FOLLOWING_SYMBOL => 80000, + self::GEOMCOLLECTION_SYMBOL => 80000, + self::GET_MASTER_PUBLIC_KEY_SYMBOL => 80000, + self::GET_SOURCE_PUBLIC_KEY_SYMBOL => 80000, + self::GROUPING_SYMBOL => 80000, + self::GROUPS_SYMBOL => 80000, + self::GTID_ONLY_SYMBOL => 80000, + self::HISTOGRAM_SYMBOL => 80000, + self::HISTORY_SYMBOL => 80000, + self::INACTIVE_SYMBOL => 80014, + self::INTERSECT_SYMBOL => 80031, + self::INVISIBLE_SYMBOL => 80000, + self::JSON_ARRAYAGG_SYMBOL => 80000, + self::JSON_OBJECTAGG_SYMBOL => 80000, + self::JSON_TABLE_SYMBOL => 80000, + self::JSON_VALUE_SYMBOL => 80021, + self::KEYRING_SYMBOL => 80024, + self::LAG_SYMBOL => 80000, + self::LAST_VALUE_SYMBOL => 80000, + self::LATERAL_SYMBOL => 80014, + self::LEAD_SYMBOL => 80000, + self::LOCKED_SYMBOL => 80000, + self::MASTER_COMPRESSION_ALGORITHM_SYMBOL => 80018, + self::MASTER_PUBLIC_KEY_PATH_SYMBOL => 80000, + self::MASTER_TLS_CIPHERSUITES_SYMBOL => 80018, + self::MASTER_ZSTD_COMPRESSION_LEVEL_SYMBOL => 80018, + self::MEMBER_SYMBOL => 80017, + self::NESTED_SYMBOL => 80000, + self::NETWORK_NAMESPACE_SYMBOL => 80017, + self::NOWAIT_SYMBOL => 80000, + self::NTH_VALUE_SYMBOL => 80000, + self::NTILE_SYMBOL => 80000, + self::NULLS_SYMBOL => 80000, + self::OF_SYMBOL => 80000, + self::OFF_SYMBOL => 80019, + self::OJ_SYMBOL => 80017, + self::OLD_SYMBOL => 80014, + self::OPTIONAL_SYMBOL => 80013, + self::ORDINALITY_SYMBOL => 80000, + self::ORGANIZATION_SYMBOL => 80011, + self::OTHERS_SYMBOL => 80000, + self::OVER_SYMBOL => 80000, + self::PASSWORD_LOCK_TIME_SYMBOL => 80019, + self::PATH_SYMBOL => 80000, + self::PERCENT_RANK_SYMBOL => 80000, + self::PERSIST_ONLY_SYMBOL => 80000, + self::PERSIST_SYMBOL => 80000, + self::PRECEDING_SYMBOL => 80000, + self::PRIVILEGE_CHECKS_USER_SYMBOL => 80018, + self::RANDOM_SYMBOL => 80018, + self::RANK_SYMBOL => 80000, + self::RECURSIVE_SYMBOL => 80000, + self::REDO_LOG_SYMBOL => 80021, + self::REFERENCE_SYMBOL => 80011, + self::REQUIRE_ROW_FORMAT_SYMBOL => 80019, + self::REQUIRE_TABLE_PRIMARY_KEY_CHECK_SYMBOL => 80019, + self::RESOURCE_SYMBOL => 80000, + self::RESPECT_SYMBOL => 80000, + self::RESTART_SYMBOL => 80011, + self::RETAIN_SYMBOL => 80014, + self::REUSE_SYMBOL => 80000, + self::RETURNING_SYMBOL => 80021, + self::ROLE_SYMBOL => 80000, + self::ROW_NUMBER_SYMBOL => 80000, + self::SECONDARY_ENGINE_ATTRIBUTE_SYMBOL => 80021, + self::SECONDARY_ENGINE_SYMBOL => 80013, + self::SECONDARY_LOAD_SYMBOL => 80013, + self::SECONDARY_SYMBOL => 80013, + self::SECONDARY_UNLOAD_SYMBOL => 80013, + self::SKIP_SYMBOL => 80000, + self::SOURCE_AUTO_POSITION_SYMBOL => 80000, + self::SOURCE_BIND_SYMBOL => 80000, + self::SOURCE_COMPRESSION_ALGORITHM_SYMBOL => 80000, + self::SOURCE_CONNECT_RETRY_SYMBOL => 80000, + self::SOURCE_CONNECTION_AUTO_FAILOVER_SYMBOL => 80000, + self::SOURCE_DELAY_SYMBOL => 80000, + self::SOURCE_HEARTBEAT_PERIOD_SYMBOL => 80000, + self::SOURCE_HOST_SYMBOL => 80000, + self::SOURCE_LOG_FILE_SYMBOL => 80000, + self::SOURCE_LOG_POS_SYMBOL => 80000, + self::SOURCE_PASSWORD_SYMBOL => 80000, + self::SOURCE_PORT_SYMBOL => 80000, + self::SOURCE_PUBLIC_KEY_PATH_SYMBOL => 80000, + self::SOURCE_RETRY_COUNT_SYMBOL => 80000, + self::SOURCE_SSL_CA_SYMBOL => 80000, + self::SOURCE_SSL_CAPATH_SYMBOL => 80000, + self::SOURCE_SSL_CERT_SYMBOL => 80000, + self::SOURCE_SSL_CIPHER_SYMBOL => 80000, + self::SOURCE_SSL_CRL_SYMBOL => 80000, + self::SOURCE_SSL_CRLPATH_SYMBOL => 80000, + self::SOURCE_SSL_KEY_SYMBOL => 80000, + self::SOURCE_SSL_SYMBOL => 80000, + self::SOURCE_SSL_VERIFY_SERVER_CERT_SYMBOL => 80000, + self::SOURCE_TLS_CIPHERSUITES_SYMBOL => 80000, + self::SOURCE_TLS_VERSION_SYMBOL => 80000, + self::SOURCE_USER_SYMBOL => 80000, + self::SOURCE_ZSTD_COMPRESSION_LEVEL_SYMBOL => 80000, + self::SRID_SYMBOL => 80000, + self::STREAM_SYMBOL => 80019, + self::SYSTEM_SYMBOL => 80000, + self::THREAD_PRIORITY_SYMBOL => 80000, + self::TIES_SYMBOL => 80000, + self::TLS_SYMBOL => 80016, + self::UNBOUNDED_SYMBOL => 80000, + self::VCPU_SYMBOL => 80000, + self::VISIBLE_SYMBOL => 80000, + self::WINDOW_SYMBOL => 80000, + self::ZONE_SYMBOL => 80022, + ); + + /** + * Identifier-like strings that may represent underscore-prefixed charset names. + * + * Includes charsets from both MySQL 5 and 8; via "SHOW CHARACTER SET"/docs: + * https://dev.mysql.com/doc/refman/5.7/en/charset-charsets.html + * https://dev.mysql.com/doc/refman/8.4/en/charset-charsets.html + * + * @TODO: Make the list respect the MySQL version. The _utf8 underscore charset + * exists only on MySQL 5, and maybe some others are version-dependant too. + * We can check this using SHOW CHARACTER SET on different MySQL versions. + */ + const UNDERSCORE_CHARSETS = array( + '_armscii8' => true, + '_ascii' => true, + '_big5' => true, + '_binary' => true, + '_cp1250' => true, + '_cp1251' => true, + '_cp1256' => true, + '_cp1257' => true, + '_cp850' => true, + '_cp852' => true, + '_cp866' => true, + '_cp932' => true, + '_dec8' => true, + '_eucjpms' => true, + '_euckr' => true, + '_gb18030' => true, + '_gb2312' => true, + '_gbk' => true, + '_geostd8' => true, + '_greek' => true, + '_hebrew' => true, + '_hp8' => true, + '_keybcs2' => true, + '_koi8r' => true, + '_koi8u' => true, + '_latin1' => true, + '_latin2' => true, + '_latin5' => true, + '_latin7' => true, + '_macce' => true, + '_macroman' => true, + '_sjis' => true, + '_swe7' => true, + '_tis620' => true, + '_ucs2' => true, + '_ujis' => true, + '_utf16' => true, + '_utf16le' => true, + '_utf32' => true, + '_utf8' => true, + '_utf8mb3' => true, + '_utf8mb4' => true, + ); + + /** + * The SQL payload to tokenize. + * + * @var string + */ + private $sql; + + /** + * The version of the MySQL server that the SQL payload is intended for. + * + * This is used to determine which tokens are valid for the given MySQL + * version, and how some tokens should be interpreted. + * + * @var int + */ + private $mysql_version; + + /** + * The SQL modes that should be considered active during tokenization. + * + * This is an integer that represents currently active SQL modes as a bitmask. + * The SQL modes are defined as "SQL_MODE_"-prefixed constants in this class. + * The list of the SQL modes isn't exhaustive, as only some affect tokenization. + * + * @var int + */ + private $sql_modes = 0; + + /** + * How many bytes from the original SQL payload have been read and tokenized. + * + * This is an internal cursor that is used to track the current position in + * the SQL payload during tokenization. When used as an index in the SQL + * payload, it points to the next byte to read. + * + * @var int + */ + private $bytes_already_read = 0; + + /** + * Byte offset in the SQL payload where current token starts. + * + * This is used to extract the token bytes after the token is processed. + * The bytes of the current token are represented by "$this->sql" in range + * from "$this->token_starts_at" to "$this->bytes_already_read - 1". + * + * @var int + */ + private $token_starts_at = 0; + + /** + * The type of the current token. + * + * When a token is successfully recognized and read, this value is set to the + * constant representing the token type. When no token was read yet, or the + * end of the SQL payload or an invalid token is reached, this value is null. + * + * @var int|null + */ + private $token_type; + + /** + * Whether the tokenizer is inside an active MySQL-specific comment. + * + * MySQL supports a special comment syntax whose content is recognized as + * a comment by most database engines, but can be treated as SQL by MySQL: + * + * 1. /*! ... - The content is treated as SQL. + * 2. /*!12345 - The content is treated as SQL when "MySQL version >= 12345". + * + * @var bool + */ + private $in_mysql_comment = false; + + /** + * @param string $sql The SQL payload to tokenize. + * @param int $mysql_version The version of the MySQL server that the SQL payload is intended for. + * @param string[] $sql_modes The SQL modes that should be considered active during tokenization. + */ + public function __construct( + string $sql, + int $mysql_version = 80038, + array $sql_modes = array() + ) { + $this->sql = $sql; + $this->mysql_version = $mysql_version; + + foreach ( $sql_modes as $sql_mode ) { + $sql_mode = strtoupper( $sql_mode ); + if ( 'HIGH_NOT_PRECEDENCE' === $sql_mode ) { + $this->sql_modes |= self::SQL_MODE_HIGH_NOT_PRECEDENCE; + } elseif ( 'PIPES_AS_CONCAT' === $sql_mode ) { + $this->sql_modes |= self::SQL_MODE_PIPES_AS_CONCAT; + } elseif ( 'IGNORE_SPACE' === $sql_mode ) { + $this->sql_modes |= self::SQL_MODE_IGNORE_SPACE; + } elseif ( 'NO_BACKSLASH_ESCAPES' === $sql_mode ) { + $this->sql_modes |= self::SQL_MODE_NO_BACKSLASH_ESCAPES; + } + } + } + + /** + * Read the next token from the SQL payload and return it as a token object. + * + * This method reads bytes from the SQL payload until a token is recognized. + * It starts from "$this->sql[ $this->bytes_already_read ]", advances the + * number of bytes read, and returns a boolean indicating whether a token + * was successfully recognized and read. When the end of the SQL payload + * or an invalid token is reached, the method returns false. + * + * @return bool Whether a token was successfully recognized and read. + */ + public function next_token(): bool { + // We already reached the end of the SQL payload or an invalid token. + // Don't attempt to read any more bytes, and bail out immediately. + if ( + self::EOF === $this->token_type + || ( null === $this->token_type && $this->bytes_already_read > 0 ) + ) { + $this->token_type = null; + return false; + } + + do { + $this->token_starts_at = $this->bytes_already_read; + $this->token_type = $this->read_next_token(); + } while ( + self::WHITESPACE === $this->token_type + || self::COMMENT === $this->token_type + || self::MYSQL_COMMENT_START === $this->token_type + || self::MYSQL_COMMENT_END === $this->token_type + ); + + // Invalid input. + if ( null === $this->token_type ) { + return false; + } + return true; + } + + /** + * Return the current token represented as a WP_MySQL_Token object. + * + * When no token was read yet, or the end of the SQL payload or an invalid + * token is reached, the method returns null. + * + * @TODO: Consider referential stability ($lexer->get_token() === $lexer->get_token()), + * or separate getters for the token type and token bytes (no token objects). + * + * @return WP_MySQL_Token|null An object representing the next recognized token or null. + */ + public function get_token(): ?WP_MySQL_Token { + if ( null === $this->token_type ) { + return null; + } + return new WP_MySQL_Token( + $this->token_type, + $this->token_starts_at, + $this->bytes_already_read - $this->token_starts_at, + $this->sql, + $this->is_sql_mode_active( self::SQL_MODE_NO_BACKSLASH_ESCAPES ) + ); + } + + /** + * Read all remaining tokens from the SQL payload and return them as an array. + * + * This method starts from the current position in the SQL payload, as marked + * by "$this->sql[ $this->bytes_already_read ]", and reads all tokens until + * the end of the SQL payload is reached, returning an array of token objects. + * + * When an invalid token is reached, the method stops and returns the partial + * sequence of valid tokens. In this case, the EOF token will not be included. + * + * This method can be used to tokenize the whole SQL payload at once, at the + * expense of storing all token objects in memory at the same time. + * + * @return WP_MySQL_Token[] An array of token objects representing the remaining tokens. + */ + public function remaining_tokens(): array { + $tokens = array(); + while ( true === $this->next_token() ) { + $token = $this->get_token(); + $tokens[] = $token; + } + return $tokens; + } + + /** + * The version of the MySQL server that the SQL payload is intended for. + * + * This represents the MySQL server version that the lexer is set up to + * consider when tokenizing the SQL payload. + * + * @return int The MySQL server version that the lexer is set up to consider. + */ + public function get_mysql_version(): int { + return $this->mysql_version; + } + + /** + * Whether an SQL mode is set to be considered as active during tokenization. + * The SQL modes are defined as "SQL_MODE_"-prefixed constants in this class. + * + * @param int $mode The SQL mode to check, an "SQL_MODE_"-prefixed constant. + * @return bool Whether the given SQL mode is active. + */ + public function is_sql_mode_active( int $mode ): bool { + return ( $this->sql_modes & $mode ) !== 0; + } + + /** + * Get the numeric token ID for a given token name. + * + * @param string $token_name The name of the token. + * @return int|null The token ID for the given token name; null when not found. + */ + public static function get_token_id( string $token_name ): ?int { + $constant_name = self::class . '::' . $token_name; + if ( ! defined( $constant_name ) ) { + return null; + } + return constant( $constant_name ); + } + + /** + * Get the name of a token for a given token ID. + * + * This method is intended to be used only for testing and debugging purposes, + * when tokens need to be presented by their names in a human-readable form. + * It should not be used in production code, as it's not performance-optimized. + * + * @param int $token_id The numeric token ID. + * @return string The token name for the given token ID; null when not found. + */ + public static function get_token_name( int $token_id ): ?string { + $reflection = new ReflectionClass( self::class ); + // Reverse the array, as some constant values in the class can conflict, + // and tokens are defined at the end of the class constant definitions. + // @TODO: Consider are more robust way to determine the token name. + // E.g., prefix all token constant names with a common prefix. + $constants = array_reverse( $reflection->getConstants() ); + $token_name = array_search( $token_id, $constants, true ); + return $token_name ? $token_name : null; + } + + private function read_next_token(): ?int { + $byte = $this->sql[ $this->bytes_already_read ] ?? null; + $next_byte = $this->sql[ $this->bytes_already_read + 1 ] ?? null; + + if ( "'" === $byte || '"' === $byte || '`' === $byte ) { + $type = $this->read_quoted_text(); + } elseif ( null !== $byte && strspn( $byte, self::DIGIT_MASK ) > 0 ) { + $type = $this->read_number(); + } elseif ( '.' === $byte ) { + if ( null !== $next_byte && strspn( $next_byte, self::DIGIT_MASK ) > 0 ) { + $type = $this->read_number(); + } else { + $this->bytes_already_read += 1; + $type = self::DOT_SYMBOL; + } + } elseif ( '=' === $byte ) { + $this->bytes_already_read += 1; + $type = self::EQUAL_OPERATOR; + } elseif ( ':' === $byte ) { + $this->bytes_already_read += 1; // Consume the ':'. + if ( '=' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '='. + $type = self::ASSIGN_OPERATOR; + } else { + $type = self::COLON_SYMBOL; + } + } elseif ( '<' === $byte ) { + $this->bytes_already_read += 1; // Consume the '<'. + if ( '=' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '='. + if ( '>' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { + $this->bytes_already_read += 1; // Consume the '>'. + $type = self::NULL_SAFE_EQUAL_OPERATOR; + } else { + $type = self::LESS_OR_EQUAL_OPERATOR; + } + } elseif ( '>' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '>'. + $type = self::NOT_EQUAL_OPERATOR; + } elseif ( '<' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '<'. + $type = self::SHIFT_LEFT_OPERATOR; + } else { + $type = self::LESS_THAN_OPERATOR; + } + } elseif ( '>' === $byte ) { + $this->bytes_already_read += 1; // Consume the '>'. + if ( '=' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '='. + $type = self::GREATER_OR_EQUAL_OPERATOR; + } elseif ( '>' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '>'. + $type = self::SHIFT_RIGHT_OPERATOR; + } else { + $type = self::GREATER_THAN_OPERATOR; + } + } elseif ( '!' === $byte ) { + $this->bytes_already_read += 1; // Consume the '!'. + if ( '=' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '='. + $type = self::NOT_EQUAL_OPERATOR; + } else { + $type = self::LOGICAL_NOT_OPERATOR; + } + } elseif ( '+' === $byte ) { + $this->bytes_already_read += 1; + $type = self::PLUS_OPERATOR; + } elseif ( '-' === $byte ) { + if ( + '-' === $next_byte + && $this->bytes_already_read + 2 < strlen( $this->sql ) + && strspn( $this->sql[ $this->bytes_already_read + 2 ], self::WHITESPACE_MASK ) > 0 + ) { + $type = $this->read_line_comment(); + } elseif ( '>' === $next_byte ) { + $this->bytes_already_read += 2; // Consume the '->'. + if ( '>' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { + $this->bytes_already_read += 1; // Consume the '>'. + if ( $this->mysql_version >= 50713 ) { + $type = self::JSON_UNQUOTED_SEPARATOR_SYMBOL; + } else { + return null; // Invalid input. + } + } else { + if ( $this->mysql_version >= 50708 ) { + $type = self::JSON_SEPARATOR_SYMBOL; + } else { + return null; // Invalid input. + } + } + } else { + $this->bytes_already_read += 1; // Consume the '-'. + $type = self::MINUS_OPERATOR; + } + } elseif ( '*' === $byte ) { + $this->bytes_already_read += 1; + if ( '/' === $next_byte && $this->in_mysql_comment ) { + $this->bytes_already_read += 1; // Consume the '/'. + $type = self::MYSQL_COMMENT_END; + $this->in_mysql_comment = false; + } else { + $type = self::MULT_OPERATOR; + } + } elseif ( '/' === $byte ) { + if ( '*' === $next_byte ) { + if ( '!' === ( $this->sql[ $this->bytes_already_read + 2 ] ?? null ) ) { + $type = $this->read_mysql_comment(); + } else { + $this->bytes_already_read += 2; // Consume the '/*'. + $this->read_comment_content(); + $type = self::COMMENT; + } + } else { + $this->bytes_already_read += 1; + $type = self::DIV_OPERATOR; + } + } elseif ( '%' === $byte ) { + $this->bytes_already_read += 1; + $type = self::MOD_OPERATOR; + } elseif ( '&' === $byte ) { + $this->bytes_already_read += 1; // Consume the '&'. + if ( '&' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '&'. + $type = self::LOGICAL_AND_OPERATOR; + } else { + $type = self::BITWISE_AND_OPERATOR; + } + } elseif ( '^' === $byte ) { + $this->bytes_already_read += 1; + $type = self::BITWISE_XOR_OPERATOR; + } elseif ( '|' === $byte ) { + $this->bytes_already_read += 1; // Consume the '|'. + if ( '|' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the '|'. + $type = $this->is_sql_mode_active( self::SQL_MODE_PIPES_AS_CONCAT ) + ? self::CONCAT_PIPES_SYMBOL + : self::LOGICAL_OR_OPERATOR; + } else { + $type = self::BITWISE_OR_OPERATOR; + } + } elseif ( '~' === $byte ) { + $this->bytes_already_read += 1; + $type = self::BITWISE_NOT_OPERATOR; + } elseif ( ',' === $byte ) { + $this->bytes_already_read += 1; + $type = self::COMMA_SYMBOL; + } elseif ( ';' === $byte ) { + $this->bytes_already_read += 1; + $type = self::SEMICOLON_SYMBOL; + } elseif ( '(' === $byte ) { + $this->bytes_already_read += 1; + $type = self::OPEN_PAR_SYMBOL; + } elseif ( ')' === $byte ) { + $this->bytes_already_read += 1; + $type = self::CLOSE_PAR_SYMBOL; + } elseif ( '{' === $byte ) { + $this->bytes_already_read += 1; + $type = self::OPEN_CURLY_SYMBOL; + } elseif ( '}' === $byte ) { + $this->bytes_already_read += 1; + $type = self::CLOSE_CURLY_SYMBOL; + } elseif ( '@' === $byte ) { + $this->bytes_already_read += 1; // Consume the '@'. + + if ( '@' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the second '@'. + $type = self::AT_AT_SIGN_SYMBOL; + } else { + /** + * Check whether the '@' marks an unquoted user-defined variable: + * https://dev.mysql.com/doc/refman/8.4/en/user-variables.html + * + * Rules: + * 1. Starts with a '@'. + * 2. Allowed following characters are ASCII a-z, A-Z, 0-9, _, ., $. + */ + $length = strspn( $this->sql, 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_.$', $this->bytes_already_read ); + if ( $length > 0 ) { + $this->bytes_already_read += $length; + $type = self::AT_TEXT_SUFFIX; + } else { + $type = self::AT_SIGN_SYMBOL; + } + } + } elseif ( '?' === $byte ) { + $this->bytes_already_read += 1; + $type = self::PARAM_MARKER; + } elseif ( '\\' === $byte ) { + $this->bytes_already_read += 1; // Consume the '\'. + if ( 'N' === $next_byte ) { + $this->bytes_already_read += 1; // Consume the 'N'. + $type = self::NULL2_SYMBOL; + } else { + return null; // Invalid input. + } + } elseif ( '#' === $byte ) { + $type = $this->read_line_comment(); + } elseif ( null !== $byte && strspn( $byte, self::WHITESPACE_MASK ) > 0 ) { + $this->bytes_already_read += strspn( $this->sql, self::WHITESPACE_MASK, $this->bytes_already_read ); + $type = self::WHITESPACE; + } elseif ( ( 'x' === $byte || 'X' === $byte || 'b' === $byte || 'B' === $byte ) && "'" === $next_byte ) { + $type = $this->read_number(); + } elseif ( ( 'n' === $byte || 'N' === $byte ) && "'" === $next_byte ) { + $this->bytes_already_read += 1; // n/N + $type = $this->read_quoted_text( "'" ); + if ( self::SINGLE_QUOTED_TEXT === $type ) { + $type = self::NCHAR_TEXT; + } + } elseif ( null === $byte ) { + $type = self::EOF; + } else { + $started_at = $this->bytes_already_read; + $type = $this->read_identifier(); + if ( self::IDENTIFIER === $type ) { + // When preceded by a dot, it is always an identifier. + if ( $started_at > 0 && '.' === $this->sql[ $started_at - 1 ] ) { + $type = self::IDENTIFIER; + } elseif ( '_' === $byte && isset( self::UNDERSCORE_CHARSETS[ strtolower( $this->get_current_token_bytes() ) ] ) ) { + $type = self::UNDERSCORE_CHARSET; + } else { + $type = $this->determine_identifier_or_keyword_type( $this->get_current_token_bytes() ); + } + } + } + return $type; + } + + private function get_current_token_bytes(): string { + return substr( + $this->sql, + $this->token_starts_at, + $this->bytes_already_read - $this->token_starts_at + ); + } + + /** + * Read an unquoted identifier. + * + * This function reads characters that are allowed in an unquoted identifier. + * An identifier cannot consist solely of digits, but this function doesn't + * ensure that explicitly, as numbers are processed before identifiers in + * the tokenization process, recognizing all digit-only sequences as numbers. + * + * Rules: + * 1. Allowed characters are ASCII a-z, A-Z, 0-9, _, $, and Unicode U+0080-U+FFFF. + * 2. Unquoted identifiers may begin with a digit but may not consist solely of digits. + * + * See: + * https://dev.mysql.com/doc/refman/8.4/en/identifiers.html + */ + private function read_identifier(): ?int { + $started_at = $this->bytes_already_read; + while ( true ) { + // First, let's try to parse an ASCII sequence. + $this->bytes_already_read += strspn( + $this->sql, + 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_$', + $this->bytes_already_read + ); + + // Check if the following byte can be part of a multibyte character + // in the range of U+0080 to U+FFFF before looking at further bytes. + // If it can't, bail out early to avoid unnecessary UTF-8 decoding. + // Identifiers are usually ASCII-only, so we can optimize for that. + $byte_1 = ord( + $this->sql[ $this->bytes_already_read ] ?? "\0" + ); + if ( $byte_1 < 0xC2 || $byte_1 > 0xEF ) { + break; + } + + // Look for a valid 2-byte UTF-8 symbol. Covers range U+0080 - U+07FF. + $byte_2 = ord( + $this->sql[ $this->bytes_already_read + 1 ] ?? "\0" + ); + if ( + $byte_1 <= 0xDF + && $byte_2 >= 0x80 && $byte_2 <= 0xBF + ) { + $this->bytes_already_read += 2; + continue; + } + + // Look for a valid 3-byte UTF-8 symbol in range U+0800 - U+FFFF. + $byte_3 = ord( + $this->sql[ $this->bytes_already_read + 2 ] ?? "\0" + ); + if ( + $byte_1 <= 0xEF + && $byte_2 >= 0x80 && $byte_2 <= 0xBF + && $byte_3 >= 0x80 && $byte_3 <= 0xBF + // Exclude surrogate range U+D800 to U+DFFF: + && ! ( 0xED === $byte_1 && $byte_2 >= 0xA0 ) + // Exclude overlong encodings: + && ! ( 0xE0 === $byte_1 && $byte_2 < 0xA0 ) + ) { + $this->bytes_already_read += 3; + continue; + } + + // Not a valid identifier character. + break; + } + + // An identifier cannot consist solely of digits, but we don't need to + // ensure that explicitly, as numbers are processed before identifiers. + + return $this->bytes_already_read - $started_at > 0 + ? self::IDENTIFIER + : null; // Invalid input. + } + + private function read_number(): ?int { + // @TODO: Support numeric-only identifier parts after "." (e.g., 1ea10.1). + + $byte = $this->sql[ $this->bytes_already_read ] ?? null; + $next_byte = $this->sql[ $this->bytes_already_read + 1 ] ?? null; + $third_byte = $this->sql[ $this->bytes_already_read + 2 ] ?? null; + + if ( + // HEX number in the form of 0xN. + ( + '0' === $byte + && 'x' === $next_byte + && null !== $third_byte + && strspn( $third_byte, self::HEX_DIGIT_MASK ) > 0 + ) + // HEX number in the form of x'N' or X'N'. + || ( ( 'x' === $byte || 'X' === $byte ) && "'" === $next_byte ) + ) { + $is_quoted = "'" === $next_byte; + $this->bytes_already_read += 2; // Consume "0x" or "x'". + $this->bytes_already_read += strspn( $this->sql, self::HEX_DIGIT_MASK, $this->bytes_already_read ); + if ( $is_quoted ) { + if ( + $this->bytes_already_read >= strlen( $this->sql ) + || "'" !== $this->sql[ $this->bytes_already_read ] + ) { + return null; // Invalid input. + } + $this->bytes_already_read += 1; // Consume the "'". + } + $type = self::HEX_NUMBER; + } elseif ( + // BIN number in the form of 0bN. + ( + '0' === $byte + && 'b' === $next_byte + && ( '0' === $third_byte || '1' === $third_byte ) + ) + // BIN number in the form of b'N' or B'N'. + || ( ( 'b' === $byte || 'B' === $byte ) && "'" === $next_byte ) + ) { + $is_quoted = "'" === $next_byte; + $this->bytes_already_read += 2; // Consume "0b" or "b'". + $this->bytes_already_read += strspn( $this->sql, '01', $this->bytes_already_read ); + if ( $is_quoted ) { + if ( + $this->bytes_already_read >= strlen( $this->sql ) + || "'" !== $this->sql[ $this->bytes_already_read ] + ) { + return null; // Invalid input. + } + $this->bytes_already_read += 1; // Consume the "'". + } + $type = self::BIN_NUMBER; + } else { + // Here, we have a sequence starting with N or .N, where N is a digit. + + // 1. Try integer first. + $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); + $type = self::INT_NUMBER; + + // 2. In case of N. or .N, it's a decimal or float number. + if ( '.' === ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { + $this->bytes_already_read += 1; + $type = self::DECIMAL_NUMBER; + $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); + } + + // 3. When exponent is present, it's a float number. + $byte = $this->sql[ $this->bytes_already_read ] ?? null; + $next_byte = $this->sql[ $this->bytes_already_read + 1 ] ?? null; + $has_exponent = + ( 'e' === $byte || 'E' === $byte ) + && null !== $next_byte + && ( + strspn( $next_byte, self::DIGIT_MASK ) > 0 + || ( + ( '+' === $next_byte || '-' === $next_byte ) + && $this->bytes_already_read + 2 < strlen( $this->sql ) + && strspn( $this->sql[ $this->bytes_already_read + 2 ], self::DIGIT_MASK ) > 0 + ) + ); + if ( $has_exponent ) { + $this->bytes_already_read += 1; // Consume the 'e' or 'E'. + $this->bytes_already_read += 1; // Consume the '+', '-', or digit. + $this->bytes_already_read += strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read ); + $type = self::FLOAT_NUMBER; + } + } + + /* + * In MySQL, when an input matches both a number and an identifier, the + * number always wins. However, if the number is followed by a non-numeric + * identifier-like character, then it is recognized as an identifier... + * Unless it's a float number, which ignores any subsequent input. + * + * Examples: + * - "1234" (integer) vs. "1234a" (identifier) + * - "0b01" (bin) vs. "0b012" (identifier) + * - "0xa1" (hex) vs. "0xa1x" (identifier) + * - "12.3" (decimal) vs. "12.3a" (identifier) + * - "1e10" (float) vs. "1e10a" (float, followed by identifier) + */ + $text = $this->get_current_token_bytes(); + $possible_identifier_prefix = + self::INT_NUMBER === $type + || ( '0' === $text[0] && ( 'b' === $text[1] || 'x' === $text[1] ) ); + + /* + * When we match some subsequent identifier bytes, it's an identifier. + * Note that the "$this->read_identifier()" method doesn't check that + * the identifier doesn't consist solely of digits. This is an advantage + * here, as we can look only at subsequent bytes instead of backtracking + * to the beginning of the number (for valid identifiers like 0b019). + */ + if ( $possible_identifier_prefix && self::IDENTIFIER === $this->read_identifier() ) { + $type = self::IDENTIFIER; + } + + // Determine integer type. + if ( self::INT_NUMBER === $type ) { + // Fast path for most integers. + $bytes = $this->get_current_token_bytes(); + if ( strlen( $bytes ) < 10 ) { + return self::INT_NUMBER; + } + + // Remove leading zeros. + $bytes = substr( $bytes, strspn( $bytes, '0' ) ); + $length = strlen( $bytes ); + + // Determine integer type based on its length and value. + if ( $length < 10 ) { + return self::INT_NUMBER; + } elseif ( 10 === $length ) { + return strcmp( $bytes, '2147483647' ) > 0 + ? self::LONG_NUMBER + : self::INT_NUMBER; + } elseif ( $length < 19 ) { + return self::LONG_NUMBER; + } elseif ( 19 === $length ) { + return strcmp( $bytes, '9223372036854775807' ) > 0 + ? self::ULONGLONG_NUMBER + : self::LONG_NUMBER; + } elseif ( 20 === $length ) { + return strcmp( $bytes, '18446744073709551615' ) > 0 + ? self::DECIMAL_NUMBER + : self::ULONGLONG_NUMBER; + } else { + return self::DECIMAL_NUMBER; + } + } + return $type; + } + + /** + * Quoted literals and identifiers: + * https://dev.mysql.com/doc/refman/8.4/en/string-literals.html + * https://dev.mysql.com/doc/refman/8.4/en/identifiers.html + * + * Rules: + * 1. Quotes can be escaped by doubling them ('', "", ``). + * 2. Backslashes escape the next character, unless NO_BACKSLASH_ESCAPES is set. + * + * @param string $quote The quote character - ', ", or `. + */ + private function read_quoted_text(): ?int { + $quote = $this->sql[ $this->bytes_already_read ]; + $this->bytes_already_read += 1; // Consume the quote. + + $no_backslash_escapes = $this->is_sql_mode_active( + self::SQL_MODE_NO_BACKSLASH_ESCAPES + ); + + // We need to look for the closing quote in a loop, as it can be escaped, + // in which case the escape sequence is consumed and the loop continues. + $at = $this->bytes_already_read; + while ( true ) { + $at += strcspn( $this->sql, $quote, $at ); + + /* + * By default, quotes can be escaped with a "\". + * When NO_BACKSLASH_ESCAPES SQL mode is active, the "\" treated as + * a regular character. + * + * The quote is escaped only when the number of preceding backslashes + * is odd - "\" is an escape sequence, "\\" is an escaped backslash, + * "\\\" is an escaped backslash and an escape sequence, and so on. + */ + if ( ! $no_backslash_escapes ) { + for ($i = 0; '\\' === $this->sql[ $at - $i - 1 ]; $i += 1); + if ( 1 === $i % 2 ) { + $at += 1; + continue; + } + } + + // Unclosed string - unexpected EOF. + if ( ( $this->sql[ $at ] ?? null ) !== $quote ) { + return null; // Invalid input. + } + + // Check if the quote is doubled. + if ( ( $this->sql[ $at + 1 ] ?? null ) === $quote ) { + $at += 2; + continue; + } + + break; + } + $at += 1; + + $this->bytes_already_read = $at; + + if ( '`' === $quote ) { + return self::BACK_TICK_QUOTED_ID; + } elseif ( '"' === $quote ) { + return self::DOUBLE_QUOTED_TEXT; + } else { + return self::SINGLE_QUOTED_TEXT; + } + } + + private function read_line_comment(): int { + $this->bytes_already_read += strcspn( $this->sql, "\r\n", $this->bytes_already_read ); + return self::COMMENT; + } + + private function read_mysql_comment(): int { + // @TODO: Consider supporting optimizer hints (/*+ ... */) or document + // that they are not supported. + // @TODO: Implement six-digit version number support (from MySQL 8.4). + + // MySQL-specific comment in one of the following forms: + // 1. /*! ... */ - The content is treated as SQL. + // 2. /*!12345 ... */ - The content is treated as SQL when "MySQL version >= 12345". + $this->bytes_already_read += 3; // Consume the '/*!'. + + // Check if the next 5 characters are digits. + $digit_count = strspn( $this->sql, self::DIGIT_MASK, $this->bytes_already_read, 5 ); + $is_version_comment = 5 === $digit_count; + + // For version comments, extract the version number. + $version = $is_version_comment + ? (int) substr( $this->sql, $this->bytes_already_read, $digit_count ) + : 0; + + if ( $this->mysql_version < $version ) { + // Version not satisfied. Treat the content as a regular comment. + $this->read_comment_content(); + return self::COMMENT; + } else { + // Version satisfied or not specified. Treat the content as SQL code. + $this->bytes_already_read += $digit_count; // Skip the version number. + $this->in_mysql_comment = true; + return self::MYSQL_COMMENT_START; + } + } + + private function read_comment_content(): void { + while ( true ) { + $this->bytes_already_read += strcspn( $this->sql, '*', $this->bytes_already_read ); + $this->bytes_already_read += 1; // Consume the '*'. + $byte = $this->sql[ $this->bytes_already_read ] ?? null; + if ( null === $byte ) { + break; + } + if ( '/' === $byte ) { + $this->bytes_already_read += 1; // Consume the '/'. + break; + } + } + } + + private function determine_identifier_or_keyword_type( string $value ): int { + $value = strtoupper( $value ); + + // Lookup the string in the token table. + $type = self::TOKENS[ $value ] ?? self::IDENTIFIER; + if ( self::IDENTIFIER === $type ) { + return self::IDENTIFIER; + } + + // Apply MySQL version specifics (positive number: >= , negative number: < ). + if ( isset( self::VERSIONS[ $type ] ) ) { + $version = self::VERSIONS[ $type ]; + if ( $this->mysql_version < $version || -$version >= $this->mysql_version ) { + return self::IDENTIFIER; + } + } + + // Apply MySQL version ranges manually. + if ( + self::MAX_STATEMENT_TIME_SYMBOL === $type + && ! ( $this->mysql_version >= 50704 && $this->mysql_version < 50708 ) + ) { + return self::IDENTIFIER; + } + + if ( + self::NONBLOCKING_SYMBOL === $type + && ! ( $this->mysql_version >= 50700 && $this->mysql_version < 50706 ) + ) { + return self::IDENTIFIER; + } + + if ( + self::REMOTE_SYMBOL === $type + && ( $this->mysql_version >= 80003 && $this->mysql_version < 80014 ) + ) { + return self::IDENTIFIER; + } + + // Determine function calls. + if ( isset( self::FUNCTIONS[ $type ] ) ) { + // Skip any whitespace character if the SQL mode says they should be ignored. + if ( $this->is_sql_mode_active( self::SQL_MODE_IGNORE_SPACE ) ) { + $this->bytes_already_read += strspn( $this->sql, self::WHITESPACE_MASK, $this->bytes_already_read ); + } + if ( '(' !== ( $this->sql[ $this->bytes_already_read ] ?? null ) ) { + return self::IDENTIFIER; + } + } + + // With "SQL_MODE_HIGH_NOT_PRECEDENCE" enabled, "NOT" needs to be emitted as a higher priority NOT2 symbol. + if ( self::NOT_SYMBOL === $type && $this->is_sql_mode_active( self::SQL_MODE_HIGH_NOT_PRECEDENCE ) ) { + $type = self::NOT2_SYMBOL; + } + + // Apply synonyms. + return self::SYNONYMS[ $type ] ?? $type; + } +} diff --git a/importer/lib/mysql-query-stream/class-wp-mysql-naive-query-stream.php b/importer/lib/mysql-query-stream/class-wp-mysql-naive-query-stream.php new file mode 100644 index 00000000..1a517b40 --- /dev/null +++ b/importer/lib/mysql-query-stream/class-wp-mysql-naive-query-stream.php @@ -0,0 +1,162 @@ +append_sql( 'SELECT id FROM users; SELECT * FROM posts;' ); + * while ( $stream->next_query() ) { + * $sql_string = $stream->get_query(); + * // Process the query. + * } + * $stream->append_sql( 'CREATE TABLE users (id INT, name VARCHAR(255));' ); + * while ( $stream->next_query() ) { + * $sql_string = $stream->get_query(); + * // Process the query. + * } + * $stream->mark_input_complete(); + * $stream->next_query(); // returns false + * + * Vendored from WordPress/sqlite-database-integration PR #264. + */ +class WP_MySQL_Naive_Query_Stream { + + private $sql_buffer = ''; + private $input_complete = false; + private $state = true; + private $last_query = false; + + const STATE_QUERY = 'valid'; + const STATE_SYNTAX_ERROR = 'syntax_error'; + const STATE_PAUSED_ON_INCOMPLETE_INPUT = 'paused_on_incomplete_input'; + const STATE_FINISHED = 'finished'; + + /** + * The maximum size of the buffer to store the SQL input. We don't + * have enough information from the lexer to distinguish between + * an incomplete input and a syntax error so we use a heuristic – + * if we've accumulated more than this amount of SQL input, we assume + * it's a syntax error. That's why this class is called a "naive" query + * stream. + */ + const MAX_SQL_BUFFER_SIZE = 1024 * 1024 * 2; + + public function __construct() {} + + public function append_sql( string $sql ) { + if($this->input_complete) { + return false; + } + $this->sql_buffer .= $sql; + $this->state = self::STATE_QUERY; + return true; + } + + public function is_paused_on_incomplete_input(): bool { + return $this->state === self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + } + + public function mark_input_complete() { + $this->input_complete = true; + } + + public function next_query() { + $this->last_query = false; + if($this->state === self::STATE_PAUSED_ON_INCOMPLETE_INPUT) { + return false; + } + + $result = $this->do_next_query(); + if(!$result && strlen($this->sql_buffer) > self::MAX_SQL_BUFFER_SIZE) { + $this->state = self::STATE_SYNTAX_ERROR; + return false; + } + return $result; + } + + private function do_next_query() { + + $query = []; + $lexer = new WP_MySQL_Lexer( $this->sql_buffer ); + while ( $lexer->next_token() ) { + $token = $lexer->get_token(); + $query[] = $token; + if ( $token->id === WP_MySQL_Lexer::SEMICOLON_SYMBOL ) { + // Got a complete query! + break; + } + } + + if(!count($query)) { + if ( $this->input_complete ) { + $this->state = self::STATE_FINISHED; + } else { + $this->state = self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + } + return false; + } + + // The last token either needs to end with a semicolon, or be the + // last token in the input. + $last_token = $query[count($query) - 1]; + if ( + $last_token->id !== WP_MySQL_Lexer::SEMICOLON_SYMBOL && + ! $this->input_complete + ) { + $this->state = self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + return false; + } + + // See if the query has any meaningful tokens. We don't want to return + // to give the caller a comment disguised as a query. + $has_meaningful_tokens = false; + foreach($query as $token) { + if ( + $token->id !== WP_MySQL_Lexer::WHITESPACE && + $token->id !== WP_MySQL_Lexer::COMMENT && + $token->id !== WP_MySQL_Lexer::MYSQL_COMMENT_START && + $token->id !== WP_MySQL_Lexer::MYSQL_COMMENT_END && + $token->id !== WP_MySQL_Lexer::EOF + ) { + $has_meaningful_tokens = true; + break; + } + } + if(!$has_meaningful_tokens) { + if ( $this->input_complete ) { + $this->state = self::STATE_FINISHED; + } else { + $this->state = self::STATE_PAUSED_ON_INCOMPLETE_INPUT; + } + return false; + } + + // Remove the query from the input buffer and return it. + $last_byte = $last_token->start + $last_token->length; + $query = substr($this->sql_buffer, 0, $last_byte); + $this->sql_buffer = substr($this->sql_buffer, $last_byte); + $this->last_query = $query; + $this->state = self::STATE_QUERY; + return true; + } + + public function get_query() { + return $this->last_query; + } + + public function get_state() { + return $this->state; + } + +} diff --git a/importer/lib/mysql-query-stream/class-wp-mysql-token.php b/importer/lib/mysql-query-stream/class-wp-mysql-token.php new file mode 100644 index 00000000..1fb25ab4 --- /dev/null +++ b/importer/lib/mysql-query-stream/class-wp-mysql-token.php @@ -0,0 +1,183 @@ +sql_mode_no_backslash_escapes_enabled = $sql_mode_no_backslash_escapes_enabled; + } + + /** + * Get the name of the token. + * + * This method is intended to be used only for testing and debugging purposes, + * when tokens need to be presented by their names in a human-readable form. + * It should not be used in production code, as it's not performance-optimized. + * + * @return string The token name. + */ + public function get_name(): string { + $name = WP_MySQL_Lexer::get_token_name( $this->id ); + if ( null === $name ) { + $name = 'UNKNOWN'; + } + return $name; + } + + /** + * Get the real unquoted value of the token. + * + * @return string The token value. + */ + public function get_value(): string { + $value = $this->get_bytes(); + if ( + WP_MySQL_Lexer::SINGLE_QUOTED_TEXT === $this->id + || WP_MySQL_Lexer::DOUBLE_QUOTED_TEXT === $this->id + || WP_MySQL_Lexer::BACK_TICK_QUOTED_ID === $this->id + ) { + // Remove bounding quotes. + $quote = $value[0]; + $value = substr( $value, 1, -1 ); + + /* + * When the NO_BACKSLASH_ESCAPES SQL mode is enabled, we only need to + * handle escaped bounding quotes, as the other characters preserve + * their literal values. + */ + if ( $this->sql_mode_no_backslash_escapes_enabled ) { + return str_replace( $quote . $quote, $quote, $value ); + } + + /** + * Unescape MySQL escape sequences. + * + * MySQL string literals use backslash as an escape character, and + * the string bounding quotes can also be escaped by being doubled. + * + * The escaping is done according to the following rules: + * + * 1. Some special character escape sequences are recognized. + * For example, "\n" is a newline character, "\0" is ASCII NULL. + * 2. A specific treatment is applied to "\%" and "\_" sequences. + * This is due to their special meaning for pattern matching. + * 3. Other backslash-prefixed characters resolve to their literal + * values. For example, "\x" represents "x", "\\" represents "\". + * + * Despite looking similar, these rules are different from the C-style + * string escaping, so we cannot use "strip(c)slashes()" in this case. + * + * See: https://dev.mysql.com/doc/refman/8.4/en/string-literals.html + */ + $backslash = chr( 92 ); + $replacements = array( + /* + * MySQL special character escape sequences. + */ + ( $backslash . '0' ) => chr( 0 ), // An ASCII NULL character (\0). + ( $backslash . "'" ) => chr( 39 ), // A single quote character ('). + ( $backslash . '"' ) => chr( 34 ), // A double quote character ("). + ( $backslash . 'b' ) => chr( 8 ), // A backspace character. + ( $backslash . 'n' ) => chr( 10 ), // A newline (linefeed) character (\n). + ( $backslash . 'r' ) => chr( 13 ), // A carriage return character (\r). + ( $backslash . 't' ) => chr( 9 ), // A tab character (\t). + ( $backslash . 'Z' ) => chr( 26 ), // An ASCII 26 (Control+Z) character. + + /* + * Normalize escaping of "%" and "_" characters. + * + * MySQL has unusual handling for "\%" and "\_" in all string literals. + * While other sequences follow the C-style escaping ("\?" is "?", etc.), + * "\%" resolves to "\%" and "\_" resolves to "\_" (unlike in C strings). + * + * This means that "\%" behaves like "\\%", and "\_" behaves like "\\_". + * To preserve this behavior, we need to add a second backslash here. + * + * From https://dev.mysql.com/doc/refman/8.4/en/string-literals.html: + * > The \% and \_ sequences are used to search for literal instances + * > of % and _ in pattern-matching contexts where they would otherwise + * > be interpreted as wildcard characters. If you use \% or \_ outside + * > of pattern-matching contexts, they evaluate to the strings \% and + * > \_, not to % and _. + */ + ( $backslash . '%' ) => $backslash . $backslash . '%', + ( $backslash . '_' ) => $backslash . $backslash . '_', + + /* + * Preserve a double backslash as-is, so that the trailing backslash + * is not consumed as the beginning of an escape sequence like "\n". + * + * Resolving "\\" to "\" will be handled in the next step, where all + * other backslash-prefixed characters resolve to their literal values. + */ + ( $backslash . $backslash ) + => $backslash . $backslash, + + /* + * The bounding quotes can also be escaped by being doubled. + */ + ( $quote . $quote ) => $quote, + ); + + /* + * Apply the replacements. + * + * It is important to use "strtr()" and not "str_replace()", because + * "str_replace()" applies replacements one after another, modifying + * intermediate changes rather than just the original string: + * + * - str_replace( [ 'a', 'b' ], [ 'b', 'c' ], 'ab' ); // 'cc' (bad) + * - strtr( 'ab', [ 'a' => 'b', 'b' => 'c' ] ); // 'bc' (good) + */ + $value = strtr( $value, $replacements ); + + /* + * A backslash with any other character represents the character itself. + * That is, \x evaluates to x, \\ evaluates to \, and \🙂 evaluates to 🙂. + */ + $preg_quoted_backslash = preg_quote( $backslash ); + $value = preg_replace( "/$preg_quoted_backslash(.)/u", '$1', $value ); + } + return $value; + } + + /** + * Get the token representation as a string. + * + * This method is intended to be used only for testing and debugging purposes, + * when tokens need to be presented in a human-readable form. It should not + * be used in production code, as it's not performance-optimized. + * + * @return string + */ + public function __toString(): string { + return $this->get_value() . '<' . $this->id . ',' . $this->get_name() . '>'; + } +} diff --git a/importer/lib/mysql-query-stream/class-wp-parser-token.php b/importer/lib/mysql-query-stream/class-wp-parser-token.php new file mode 100644 index 00000000..b7726189 --- /dev/null +++ b/importer/lib/mysql-query-stream/class-wp-parser-token.php @@ -0,0 +1,77 @@ +id = $id; + $this->start = $start; + $this->length = $length; + $this->input = $input; + } + + /** + * Get the raw bytes of the token from the input. + * + * @return string The token bytes. + */ + public function get_bytes(): string { + return substr( $this->input, $this->start, $this->length ); + } + + /** + * Get the real unquoted value of the token. + * + * @return string The token value. + */ + public function get_value(): string { + return $this->get_bytes(); + } +} diff --git a/importer/lib/mysql-query-stream/load.php b/importer/lib/mysql-query-stream/load.php new file mode 100644 index 00000000..86ce627c --- /dev/null +++ b/importer/lib/mysql-query-stream/load.php @@ -0,0 +1,13 @@ +2000,'rules_names'=>['query','%f1','%f2','%f3','simpleStatement','%f4','%f5','%f6','%f7','alterStatement','%f8','%f9','%f10','alterInstance','%f11','%f12','%f13','%f14','%f15','%f16','%f17','%f18','%f19','%f20','%f21','%f22','%f23','%f24','alterDatabase','%f25','%f26','%f27','%f28','%f29','%f30','%f31','%f32','%f33','%f34','alterEvent','%f35','%f36','%f37','%f38','%f39','%f40','%f41','%f42','%f43','%f44','alterLogfileGroup','%f45','alterLogfileGroupOptions','%f46','%f47','alterLogfileGroupOption','alterServer','%f48','%f49','%f50','alterTable','%f51','%f52','%f53','%f54','alterTableActions','%f55','%f56','%f57','alterCommandList','%f58','%f59','alterCommandsModifierList','%f60','standaloneAlterCommands','%f61','%f62','%f63','%f64','%f65','%f66','%f67','alterPartition','%f68','%f69','%f70','%f71','%f72','alterList','%f73','%f74','%f75','alterCommandsModifier','%f76','%f77','%f78','%f79','alterListItem','%f80','%f81','%f82','%f83','%f84','%f85','%f86','%f87','%f88','%f89','%f90','%f91','%f92','%f93','%f94','%f95','%f96','%f97','%f98','%f99','%f100','%f101','%f102','place','restrict','%f103','%f104','alterOrderList','%f105','%f106','alterAlgorithmOption','%f107','alterLockOption','%f108','%f109','%f110','indexLockAndAlgorithm','withValidation','%f111','%f112','removePartitioning','allOrPartitionNameList','alterTablespace','%f113','%f114','%f115','%f116','%f117','%f118','%f119','%f120','%f121','%f122','%f123','%f124','alterUndoTablespace','%f125','%f126','undoTableSpaceOptions','%f127','undoTableSpaceOption','%f128','alterTablespaceOptions','%f129','alterTablespaceOption','%f130','changeTablespaceOption','%f131','%f132','alterView','%f133','viewTail','%f134','viewSelect','%f135','viewCheckOption','%f136','createStatement','%f137','%f138','%f139','%f140','%f141','%f142','createDatabase','createDatabaseOption','%f143','%f144','createTable','%f145','%f146','%f147','%f148','%f149','%f150','%f151','tableElementList','%f152','tableElement','%f153','%f154','duplicateAsQueryExpression','%f155','queryExpressionOrParens','%f156','createRoutine','%f157','%f158','%f159','createProcedure','%f160','%f161','%f162','%f163','%f164','%f165','createFunction','%f166','%f167','%f168','%f169','%f170','%f171','createUdf','%f172','routineCreateOption','%f173','routineAlterOptions','routineOption','%f174','%f175','createIndex','%f176','%f177','%f178','%f179','%f180','%f181','%f182','%f183','%f184','%f185','%f186','indexNameAndType','%f187','%f188','createIndexTarget','%f189','createLogfileGroup','%f190','%f191','logfileGroupOptions','%f192','logfileGroupOption','createServer','%f193','serverOptions','%f194','serverOption','%f195','%f196','createTablespace','%f197','createUndoTablespace','tsDataFileName','%f198','%f199','%f200','tsDataFile','%f201','tablespaceOptions','%f202','tablespaceOption','%f203','%f204','%f205','tsOptionInitialSize','tsOptionUndoRedoBufferSize','%f206','tsOptionAutoextendSize','tsOptionMaxSize','tsOptionExtentSize','tsOptionNodegroup','%f207','tsOptionEngine','tsOptionEngineAttribute','tsOptionWait','%f208','tsOptionComment','tsOptionFileblockSize','tsOptionEncryption','%f209','createView','viewReplaceOrAlgorithm','viewAlgorithm','%f210','viewSuid','%f211','%f212','createTrigger','%f213','%f214','%f215','%f216','triggerFollowsPrecedesClause','%f217','%f218','%f219','%f220','%f221','createEvent','%f222','%f223','%f224','%f225','%f226','createRole','%f227','createSpatialReference','srsAttribute','dropStatement','%f228','%f229','%f230','%f231','%f232','dropDatabase','dropEvent','dropFunction','dropProcedure','dropIndex','%f233','dropLogfileGroup','%f234','%f235','%f236','dropLogfileGroupOption','dropServer','%f237','dropTable','%f238','%f239','%f240','dropTableSpace','%f241','%f242','%f243','dropTrigger','%f244','dropView','%f245','dropRole','dropSpatialReference','dropUndoTablespace','%f246','renameTableStatement','%f247','%f248','renamePair','%f249','truncateTableStatement','importStatement','%f250','callStatement','%f251','%f252','%f253','%f254','deleteStatement','%f255','%f256','%f257','%f258','%f259','%f260','%f261','%f262','%f263','%f264','%f265','partitionDelete','%f266','deleteStatementOption','doStatement','%f267','%f268','%f269','handlerStatement','%f270','%f271','%f272','%f273','handlerReadOrScan','%f274','%f275','%f276','%f277','%f278','%f279','%f280','%f281','%f282','insertStatement','%f283','%f284','%f285','%f286','%f287','%f288','%f289','insertLockOption','%f290','insertFromConstructor','%f291','%f292','%f293','fields','%f294','insertValues','%f295','insertQueryExpression','%f296','%f297','valueList','%f298','%f299','values','%f300','%f301','%f302','valuesReference','insertUpdateList','%f303','%f304','%f305','%f306','%f307','%f308','%f309','loadStatement','%f310','%f311','dataOrXml','xmlRowsIdentifiedBy','%f312','%f313','%f314','loadDataFileTail','%f315','%f316','%f317','%f318','loadDataFileTargetList','%f319','fieldOrVariableList','%f320','%f321','%f322','%f323','replaceStatement','%f324','%f325','%f326','selectStatement','selectStatementWithInto','%f327','%f328','queryExpression','%f329','%f330','%f331','%f332','%f333','%f334','queryExpressionBody','%f335','%f336','%f337','%f338','%f339','queryTerm','%f340','%f341','%f342','%f343','queryExpressionParens','queryPrimary','%f344','%f345','%f346','%f347','%f348','%f349','%f350','%f351','querySpecification','%f352','%f353','subquery','querySpecOption','limitClause','simpleLimitClause','%f354','limitOptions','%f355','%f356','limitOption','%f357','intoClause','%f358','%f359','%f360','%f361','%f362','%f363','procedureAnalyseClause','%f364','%f365','%f366','havingClause','%f367','windowClause','%f368','windowDefinition','windowSpec','%f369','%f370','%f371','%f372','%f373','windowSpecDetails','%f374','%f375','%f376','%f377','windowFrameClause','windowFrameUnits','windowFrameExtent','windowFrameStart','windowFrameBetween','windowFrameBound','windowFrameExclusion','%f378','%f379','%f380','withClause','%f381','commonTableExpression','%f382','groupByClause','olapOption','%f383','orderClause','direction','fromClause','%f384','%f385','tableReferenceList','%f386','%f387','tableValueConstructor','%f388','explicitTable','rowValueExplicit','selectOption','%f389','%f390','%f391','lockingClauseList','%f392','%f393','lockingClause','%f394','%f395','%f396','%f397','lockStrengh','%f398','lockedRowAction','%f399','selectItemList','%f400','%f401','%f402','selectItem','selectAlias','%f403','whereClause','%f404','tableReference','%f405','%f406','%f407','escapedTableReference','%f408','joinedTable','%f409','%f410','%f411','%f412','naturalJoinType','%f413','%f414','innerJoinType','%f415','outerJoinType','%f416','tableFactor','%f417','%f418','singleTable','singleTableParens','%f419','%f420','derivedTable','%f421','%f422','%f423','tableReferenceListParens','%f424','tableFunction','%f425','columnsClause','%f426','%f427','%f428','%f429','jtColumn','%f430','%f431','%f432','%f433','onEmptyOrError','onEmpty','onError','jtOnResponse','setOperationOption','%f434','tableAlias','%f435','%f436','%f437','indexHintList','%f438','%f439','indexHint','indexHintType','keyOrIndex','%f440','constraintKeyType','indexHintClause','%f441','%f442','indexList','%f443','indexListElement','%f444','%f445','updateStatement','%f446','%f447','transactionOrLockingStatement','%f448','%f449','%f450','%f451','transactionStatement','%f452','%f453','%f454','beginWork','%f455','transactionCharacteristicList','%f456','transactionCharacteristic','%f457','%f458','savepointStatement','%f459','%f460','%f461','%f462','%f463','%f464','%f465','lockStatement','%f466','%f467','%f468','%f469','%f470','lockItem','lockOption','xaStatement','%f471','%f472','%f473','%f474','%f475','%f476','%f477','%f478','%f479','%f480','xaConvert','%f481','%f482','%f483','%f484','xid','%f485','%f486','%f487','%f488','replicationStatement','%f489','%f490','%f491','%f492','%f493','%f494','%f495','%f496','%f497','%f498','%f499','resetOption','%f500','masterResetOptions','%f501','%f502','%f503','%f504','replicationLoad','%f505','changeMaster','%f506','changeMasterOptions','%f507','masterOption','privilegeCheckDef','tablePrimaryKeyCheckDef','masterTlsCiphersuitesDef','masterFileDef','%f508','serverIdList','%f509','%f510','%f511','%f512','%f513','changeReplication','%f514','%f515','%f516','%f517','changeReplicationSourceOptions','%f518','replicationSourceOption','%f519','%f520','%f521','%f522','%f523','%f524','%f525','%f526','%f527','%f528','%f529','%f530','%f531','%f532','%f533','%f534','%f535','%f536','%f537','%f538','%f539','%f540','%f541','%f542','%f543','%f544','%f545','%f546','%f547','%f548','%f549','filterDefinition','%f550','filterDbList','%f551','%f552','filterTableList','%f553','%f554','filterStringList','%f555','filterWildDbTableString','%f556','filterDbPairList','%f557','%f558','%f559','slave','%f560','%f561','slaveUntilOptions','%f562','%f563','%f564','%f565','%f566','slaveConnectionOptions','%f567','%f568','%f569','%f570','%f571','%f572','%f573','%f574','%f575','%f576','slaveThreadOptions','%f577','slaveThreadOption','groupReplication','%f578','preparedStatement','%f579','%f580','%f581','executeStatement','%f582','%f583','executeVarList','%f584','cloneStatement','%f585','%f586','%f587','%f588','%f589','%f590','%f591','dataDirSSL','ssl','accountManagementStatement','%f592','%f593','%f594','alterUser','%f595','%f596','alterUserTail','%f597','%f598','%f599','%f600','%f601','%f602','userFunction','createUser','%f603','%f604','createUserTail','%f605','%f606','%f607','%f608','%f609','%f610','%f611','%f612','defaultRoleClause','%f613','%f614','%f615','requireClause','%f616','%f617','%f618','connectOptions','%f619','accountLockPasswordExpireOptions','%f620','%f621','%f622','%f623','%f624','%f625','%f626','%f627','%f628','%f629','%f630','dropUser','%f631','%f632','grant','%f633','%f634','%f635','%f636','%f637','%f638','%f639','%f640','%f641','%f642','%f643','%f644','grantTargetList','%f645','%f646','grantOptions','%f647','%f648','%f649','exceptRoleList','withRoles','%f650','%f651','%f652','grantAs','versionedRequireClause','%f653','%f654','renameUser','%f655','%f656','%f657','revoke','%f658','%f659','%f660','%f661','%f662','%f663','%f664','%f665','%f666','%f667','%f668','onTypeTo','%f669','%f670','%f671','%f672','aclType','%f673','roleOrPrivilegesList','%f674','%f675','%f676','roleOrPrivilege','%f677','%f678','%f679','%f680','%f681','%f682','%f683','%f684','%f685','%f686','%f687','grantIdentifier','%f688','%f689','%f690','%f691','requireList','%f692','%f693','requireListElement','grantOption','%f694','setRole','%f695','%f696','%f697','%f698','roleList','%f699','%f700','role','%f701','%f702','%f703','tableAdministrationStatement','%f704','%f705','%f706','%f707','%f708','%f709','%f710','%f711','%f712','%f713','histogram','%f714','%f715','checkOption','%f716','repairType','%f717','%f718','installUninstallStatment','%f719','%f720','installOptionType','%f721','installSetValue','%f722','%f723','installSetValueList','%f724','setStatement','%f725','startOptionValueList','%f726','%f727','%f728','%f729','%f730','%f731','%f732','%f733','%f734','%f735','%f736','transactionCharacteristics','%f737','%f738','transactionAccessMode','%f739','isolationLevel','%f740','%f741','%f742','optionValueListContinued','%f743','optionValueNoOptionType','%f744','%f745','optionValue','%f746','setSystemVariable','startOptionValueListFollowingOptionType','optionValueFollowingOptionType','setExprOrDefault','%f747','%f748','%f749','showStatement','%f750','%f751','%f752','%f753','%f754','%f755','%f756','%f757','%f758','%f759','%f760','%f761','%f762','%f763','%f764','%f765','%f766','%f767','%f768','%f769','%f770','%f771','%f772','%f773','%f774','%f775','%f776','%f777','%f778','%f779','%f780','%f781','%f782','%f783','showCommandType','%f784','nonBlocking','%f785','%f786','fromOrIn','inDb','profileType','%f787','%f788','otherAdministrativeStatement','%f789','%f790','%f791','%f792','%f793','%f794','keyCacheListOrParts','%f795','keyCacheList','%f796','%f797','assignToKeycache','assignToKeycachePartition','%f798','cacheKeyList','keyUsageElement','%f799','keyUsageList','%f800','%f801','flushOption','%f802','%f803','%f804','logType','%f805','flushTables','%f806','%f807','%f808','flushTablesOptions','%f809','%f810','preloadTail','%f811','%f812','preloadList','%f813','%f814','preloadKeys','%f815','adminPartition','resourceGroupManagement','%f816','%f817','%f818','createResourceGroup','%f819','%f820','resourceGroupVcpuList','%f821','%f822','vcpuNumOrRange','%f823','resourceGroupPriority','resourceGroupEnableDisable','%f824','alterResourceGroup','%f825','setResourceGroup','%f826','%f827','threadIdList','%f828','dropResourceGroup','utilityStatement','%f829','%f830','describeStatement','%f831','%f832','%f833','explainStatement','%f834','%f835','%f836','%f837','%f838','%f839','%f840','explainableStatement','%f841','%f842','%f843','helpCommand','useCommand','restartServer','%f844','expr','%f845','%f846','%f847','%f848','%f849','%f850','%f851','%f852','%f853','%f854','boolPri','%f855','%f856','compOp','%f857','predicate','%f858','%f859','%f860','%f861','predicateOperations','%f862','%f863','%f864','bitExpr','%f865','%f866','%f867','%f868','%f869','%f870','simpleExpr','%f871','%f872','%f873','%f874','%f875','%f876','%f877','%f878','%f879','%f880','%f881','%f882','%f883','%f884','%f885','%f886','%f887','%f888','%f889','%f890','%f891','%f892','arrayCast','%f893','jsonOperator','%f894','%f895','%f896','%f897','%f898','%f899','%f900','%f901','%f902','%f903','%f904','%f905','%f906','%f907','%f908','%f909','sumExpr','%f910','%f911','%f912','%f913','%f914','%f915','%f916','%f917','%f918','%f919','%f920','%f921','%f922','%f923','%f924','%f925','%f926','%f927','%f928','%f929','%f930','%f931','%f932','%f933','%f934','%f935','%f936','%f937','groupingOperation','%f938','%f939','%f940','windowFunctionCall','%f941','%f942','%f943','%f944','%f945','windowingClause','%f946','%f947','leadLagInfo','%f948','%f949','nullTreatment','%f950','%f951','jsonFunction','inSumExpr','identListArg','%f952','identList','%f953','%f954','fulltextOptions','%f955','%f956','%f957','%f958','%f959','%f960','%f961','%f962','runtimeFunctionCall','%f963','%f964','%f965','%f966','%f967','%f968','%f969','%f970','%f971','%f972','%f973','%f974','%f975','%f976','%f977','%f978','%f979','%f980','%f981','%f982','geometryFunction','%f983','%f984','timeFunctionParameters','fractionalPrecision','%f985','weightStringLevels','%f986','%f987','%f988','%f989','weightStringLevelListItem','%f990','%f991','%f992','dateTimeTtype','trimFunction','%f993','%f994','%f995','substringFunction','%f996','%f997','%f998','%f999','%f1000','%f1001','functionCall','%f1002','udfExprList','%f1003','udfExpr','variable','userVariable','%f1004','%f1005','systemVariable','internalVariableName','%f1006','%f1007','%f1008','whenExpression','thenExpression','elseExpression','%f1009','%f1010','%f1011','%f1012','castType','%f1013','%f1014','%f1015','%f1016','%f1017','exprList','%f1018','charset','notRule','not2Rule','interval','%f1019','intervalTimeStamp','exprListWithParentheses','exprWithParentheses','simpleExprWithParentheses','%f1020','orderList','%f1021','orderExpression','%f1022','groupList','%f1023','groupingExpression','channel','%f1024','compoundStatement','returnStatement','ifStatement','%f1025','ifBody','%f1026','thenStatement','%f1027','compoundStatementList','%f1028','%f1029','%f1030','caseStatement','%f1031','elseStatement','%f1032','labeledBlock','unlabeledBlock','label','%f1033','%f1034','beginEndBlock','labeledControl','unlabeledControl','loopBlock','whileDoBlock','repeatUntilBlock','%f1035','spDeclarations','%f1036','spDeclaration','%f1037','variableDeclaration','%f1038','conditionDeclaration','spCondition','%f1039','sqlstate','%f1040','handlerDeclaration','%f1041','%f1042','handlerCondition','cursorDeclaration','iterateStatement','leaveStatement','%f1043','getDiagnostics','%f1044','%f1045','%f1046','%f1047','%f1048','%f1049','%f1050','signalAllowedExpr','statementInformationItem','%f1051','%f1052','conditionInformationItem','%f1053','%f1054','signalInformationItemName','%f1055','signalStatement','%f1056','%f1057','%f1058','%f1059','%f1060','%f1061','resignalStatement','%f1062','%f1063','%f1064','%f1065','signalInformationItem','cursorOpen','cursorClose','%f1066','cursorFetch','%f1067','%f1068','%f1069','%f1070','schedule','%f1071','%f1072','columnDefinition','checkOrReferences','%f1073','checkConstraint','constraintEnforcement','%f1074','tableConstraintDef','%f1075','%f1076','%f1077','%f1078','%f1079','%f1080','%f1081','constraintName','fieldDefinition','%f1082','%f1083','%f1084','%f1085','%f1086','%f1087','%f1088','%f1089','%f1090','%f1091','%f1092','%f1093','%f1094','columnAttribute','%f1095','%f1096','%f1097','%f1098','%f1099','%f1100','%f1101','columnFormat','storageMedia','gcolAttribute','%f1102','%f1103','%f1104','references','%f1105','%f1106','%f1107','%f1108','%f1109','%f1110','%f1111','deleteOption','%f1112','%f1113','keyList','%f1114','keyPart','%f1115','keyListWithExpression','%f1116','keyPartOrExpression','keyListVariants','%f1117','%f1118','indexType','%f1119','indexOption','commonIndexOption','%f1120','visibility','indexTypeClause','%f1121','fulltextIndexOption','spatialIndexOption','dataTypeDefinition','%f1122','%f1123','%f1124','%f1125','dataType','%f1126','%f1127','%f1128','%f1129','%f1130','%f1131','%f1132','%f1133','%f1134','%f1135','%f1136','nchar','realType','fieldLength','%f1137','%f1138','fieldOptions','%f1139','%f1140','charsetWithOptBinary','%f1141','ascii','unicode','wsNumCodepoints','typeDatetimePrecision','charsetName','%f1142','collationName','%f1143','%f1144','%f1145','createTableOptions','%f1146','%f1147','createTableOptionsSpaceSeparated','%f1148','createTableOption','%f1149','%f1150','%f1151','%f1152','%f1153','%f1154','%f1155','%f1156','%f1157','%f1158','%f1159','%f1160','%f1161','%f1162','%f1163','ternaryOption','%f1164','defaultCollation','defaultEncryption','defaultCharset','%f1165','%f1166','%f1167','partitionClause','%f1168','%f1169','%f1170','%f1171','partitionTypeDef','%f1172','%f1173','%f1174','subPartitions','%f1175','%f1176','partitionKeyAlgorithm','%f1177','%f1178','partitionDefinitions','%f1179','%f1180','%f1181','%f1182','partitionDefinition','%f1183','%f1184','%f1185','%f1186','%f1187','%f1188','partitionValuesIn','%f1189','partitionOption','%f1190','%f1191','subpartitionDefinition','%f1192','partitionValueItemListParen','%f1193','partitionValueItem','definerClause','ifExists','ifNotExists','%f1194','procedureParameter','%f1195','functionParameter','collate','typeWithOptCollate','schemaIdentifierPair','%f1196','viewRefList','%f1197','%f1198','updateList','%f1199','updateElement','%f1200','charsetClause','%f1201','fieldsClause','%f1202','fieldTerm','%f1203','linesClause','lineTerm','%f1204','%f1205','userList','%f1206','%f1207','createUserList','%f1208','%f1209','alterUserList','%f1210','%f1211','createUserEntry','%f1212','%f1213','%f1214','%f1215','%f1216','%f1217','%f1218','%f1219','%f1220','%f1221','%f1222','alterUserEntry','%f1223','%f1224','%f1225','%f1226','%f1227','%f1228','%f1229','%f1230','%f1231','%f1232','%f1233','retainCurrentPassword','discardOldPassword','replacePassword','%f1234','userIdentifierOrText','%f1235','%f1236','user','likeClause','likeOrWhere','onlineOption','noWriteToBinLog','usePartition','%f1237','fieldIdentifier','%f1238','%f1239','%f1240','columnInternalRef','%f1241','columnInternalRefList','%f1242','columnRef','insertIdentifier','indexName','indexRef','%f1243','tableWild','%f1244','schemaName','schemaRef','procedureName','procedureRef','functionName','functionRef','triggerName','triggerRef','viewName','%f1245','viewRef','%f1246','tablespaceName','tablespaceRef','logfileGroupName','logfileGroupRef','eventName','eventRef','udfName','serverName','serverRef','engineRef','tableName','%f1247','filterTableRef','%f1248','tableRefWithWildcard','%f1249','%f1250','%f1251','tableRef','%f1252','%f1253','tableRefList','%f1254','%f1255','tableAliasRefList','%f1256','parameterName','labelIdentifier','labelRef','roleIdentifier','roleRef','pluginRef','componentRef','resourceGroupRef','windowName','pureIdentifier','%f1257','%f1258','identifier','%f1259','identifierList','%f1260','identifierListWithParentheses','qualifiedIdentifier','%f1261','simpleIdentifier','%f1262','%f1263','dotIdentifier','ulong_number','real_ulong_number','ulonglong_number','real_ulonglong_number','%f1264','%f1265','literal','%f1266','signedLiteral','%f1267','stringList','%f1268','textStringLiteral','%f1269','textString','textStringHash','%f1270','%f1271','textLiteral','%f1272','textStringNoLinebreak','%f1273','textStringLiteralList','%f1274','numLiteral','boolLiteral','nullLiteral','temporalLiteral','floatOptions','standardFloatOptions','precision','textOrIdentifier','lValueIdentifier','roleIdentifierOrText','sizeNumber','parentheses','equal','optionType','varIdentType','setVarIdentType','identifierKeyword','%f1275','%f1276','%f1277','%f1278','%f1279','identifierKeywordsAmbiguous1RolesAndLabels','identifierKeywordsAmbiguous2Labels','labelKeyword','%f1280','%f1281','%f1282','identifierKeywordsAmbiguous3Roles','identifierKeywordsUnambiguous','%f1283','%f1284','%f1285','roleKeyword','%f1286','%f1287','%f1288','lValueKeyword','identifierKeywordsAmbiguous4SystemVariables','roleOrIdentifierKeyword','%f1289','%f1290','%f1291','roleOrLabelKeyword','%f1292','%f1293','%f1294','%f1295','%f1296','%f1297','%f1298'],'grammar'=>[[[-1],[2001,2003]],[[2004],[2668]],[[-1],[0]],[[755,2002],[-1]],[[2009],[2175],[2318],[2353],[2358],[2005],[2361],[2366],[2381],[2385],[2400],[2437],[2457],[2461],[2656],[2659],[2712],[2829],[2006],[2848],[2991],[3010],[3020],[3057],[2007],[3102],[3168],[2008],[3489],[3496]],[[2359]],[[2838]],[[3145]],[[3472]],[[11,2012]],[[2153]],[[2225],[0]],[[2060],[2028],[422,3783,2011],[206,3785,2011],[2167],[2039],[2140],[2010],[2050],[2056],[2013]],[[2026]],[[33]],[[844],[2014]],[[2017],[0]],[[373,480,383,165]],[[451,845,2016]],[[2020],[0]],[[373,480,383,165]],[[451,845,200,57,3830,2019]],[[156],[140]],[[2022,844,846]],[[451,847]],[[482,2015,316,265],[2018],[2021],[2023],[2024]],[[244,2025]],[[3781],[0]],[[109,2027,2031]],[[615,112,139,357]],[[2183,2030],[2183]],[[2030],[2029]],[[3690],[0]],[[2040],[0]],[[2042],[0]],[[2043],[0]],[[2046],[0]],[[2047],[0]],[[2048],[0]],[[2032,170,3797,2033,2034,2035,2036,2037,2038]],[[383,490,3510]],[[371],[0]],[[383,79,2041,418]],[[453,590,3830]],[[2045],[0]],[[383,514]],[[156],[140,2044]],[[75,3859]],[[147,3425]],[[2052],[0]],[[288,217,3795,4,603,3859,2049]],[[2054,2051],[2054],[0]],[[2055,2051]],[[750],[0]],[[2053,2055]],[[2274],[2282],[2284]],[[503,3800,2254]],[[3761],[0]],[[2062],[0]],[[2065],[0]],[[2057,2058,574,3810,2059]],[[232]],[[2061]],[[2066],[0]],[[2067],[0]],[[2063,2074],[2069,2064],[3653],[2138]],[[2072,750]],[[3653],[2138]],[[2070],[0]],[[2072,2068],[2088]],[[750,2088]],[[2073,2071],[2073],[0]],[[2092,2071]],[[750,2092]],[[141,572],[234,572],[2082],[2076]],[[722],[723]],[[2075]],[[3762],[0]],[[3005,2078],[3005],[0]],[[3007,2079],[3007],[0]],[[2084],[0]],[[2135],[0]],[[4,405,2077,2083],[148,405,3832],[438,405,2077,2139],[388,405,2077,2139,2077],[14,405,2077,2139],[62,405,2139,2078],[455,405,2077,2139,2079],[67,405,2077,3842],[597,405,2139],[454,405,2077,2080],[172,405,3830,645,574,3810,2081],[2085],[2086]],[[3668],[404,3842]],[[3832,248,3668]],[[141,405,2139,572]],[[234,405,2139,572]],[[2091,2087],[2091],[0]],[[2089,2087]],[[2097],[3627]],[[2097],[2092],[3627]],[[750,2090]],[[2128],[2130],[2135]],[[72],[0]],[[2121],[0]],[[2116],[0]],[[3697],[0]],[[4,2093,2099],[4,3519],[55,2093,3765,3830,3528,2094],[348,2093,3765,3528,2094],[148,2107],[140,263],[156,263],[11,2093,3765,2111],[2112],[2113],[2114],[2115],[453,2095,3802],[2117],[94,590,3406,2119,2096],[198],[393,45,2125],[2120]],[[3514],[0]],[[3830,3528,2098,2094],[753,2194,748]],[[3765]],[[3765],[0]],[[2101]],[[2100],[2102]],[[62,3830]],[[86,3830]],[[2122],[0]],[[2093,3765,2106],[199,265,2103],[420,265],[2645,3776],[2104],[2105]],[[3413]],[[2108],[3849]],[[506,3582]],[[506,128,2109],[148,128],[2110]],[[11,236,3776,3582]],[[11,62,3830,3517]],[[11,86,3830,3517]],[[453,72,3765,590,3830]],[[590],[17]],[[453,2645,3776,590,3775]],[[128]],[[2118],[3618]],[[615,403]],[[6,3830],[191]],[[471],[49]],[[2551],[0]],[[2126,2124],[2126],[0]],[[3837,2123,2124]],[[750,3837,2123]],[[763],[0]],[[9,2127,2129]],[[128],[3830]],[[287,2127,2131]],[[128],[3830]],[[2130],[0]],[[2128],[0]],[[2128,2132],[2130,2133]],[[2137]],[[645],[646]],[[2136,625]],[[452,403]],[[10],[3832]],[[572,3793,2151]],[[4],[148]],[[2143,2142],[2143],[0]],[[2053,2164]],[[2145],[0]],[[2164,2142]],[[434],[436]],[[55,111,3859,2144],[2146],[371,1]],[[2147]],[[2160]],[[2160],[0]],[[2141,111,3859,2150],[2148],[453,590,3830],[2149]],[[2156],[0]],[[605,572,3793,506,2154,2152]],[[724],[725]],[[2157,2155],[2157],[0]],[[2158,2155]],[[2053,2158]],[[2282]],[[2161,2159],[2161],[0]],[[2162,2159]],[[2053,2162]],[[238,2127,3875],[2277],[2278],[2282],[2163],[2284],[2288]],[[2283]],[[238,2127,3875],[2277],[2278]],[[2292],[0]],[[2294],[0]],[[2165,2032,2166,636,3790,2169]],[[3771],[0]],[[2168,17,2171]],[[2173],[0]],[[2201,2170]],[[2174],[0]],[[645,2172,62,391]],[[50],[284]],[[97,2179]],[[2314]],[[2316]],[[2261]],[[2182],[2186],[2214],[2207],[2221],[2246],[2290],[2297],[2229],[2252],[2259],[2308],[2176],[2177],[2178]],[[3692],[0]],[[2183,2181],[2183],[0]],[[109,2180,3780,2181]],[[3649],[3647],[2184]],[[3648]],[[577],[0]],[[2185,574,2180,3802,2192]],[[2188],[0]],[[753,2194,748]],[[3624],[0]],[[3653],[0]],[[2199],[0]],[[275,3810],[753,275,3810,748],[2187,2189,2190,2191]],[[2195,2193],[2195],[0]],[[2196,2193]],[[750,2196]],[[3513],[3519]],[[2200],[0]],[[17],[0]],[[2197,2198,2201]],[[458],[232]],[[2465],[2483]],[[755],[0]],[[97,2204,2202,-1]],[[2207],[2214],[2221]],[[2212],[0]],[[2223,2206],[2223],[0]],[[2032,422,2209,3782,753,2205,748,2206,3425]],[[2180]],[[2208]],[[2211,2210],[2211],[0]],[[750,3694]],[[3694,2210]],[[2219],[0]],[[2032,206,2216,3784,753,2213,748,474,3698,2206,3425]],[[2180]],[[2215]],[[2218,2217],[2218],[0]],[[750,3696]],[[3696,2217]],[[8],[0]],[[2220,206,3798,474,2222,520,3859]],[[556],[249],[437],[126]],[[2226],[2041,137]],[[2223,2224],[2223]],[[2224]],[[75,3859],[267,537],[373,537],[90,537],[433,537,112],[347,537,112],[537,496,2227]],[[130],[250]],[[2134],[0]],[[2057,2238,2228]],[[3583],[0]],[[3775,2230]],[[2241],[0]],[[2231],[2232]],[[609],[0]],[[3579,2235],[3579],[0]],[[3585,2236],[3585],[0]],[[3586,2237],[3586],[0]],[[2234,236,2233,2244,2235],[205,236,3775,2244,2236],[523,236,3775,2244,2237]],[[3775],[0]],[[2243],[0]],[[2239,2240]],[[621],[599]],[[2242,3577]],[[383,3810,3574]],[[2249],[0]],[[288,217,3794,4,2247,3859,2245]],[[603],[440]],[[2250,2248],[2250],[0]],[[2251,2248]],[[2053,2251]],[[2274],[2275],[2280],[2282],[2284],[2286]],[[503,3799,199,112,648,3872,2254]],[[2255,2253],[2255],[0]],[[390,753,2256,2253,748]],[[750,2256]],[[224,3859],[109,3859],[618,3859],[406,3859],[519,3859],[398,3859],[413,3841]],[[2260],[0]],[[2268],[0]],[[572,3792,2262,2257,2258]],[[620,288,217,3795]],[[605,572,3792,4,2266,2152]],[[2265],[4,2266]],[[2264],[0]],[[4,2266]],[[2263]],[[111,3859]],[[2269,2267],[2269],[0]],[[2270,2267]],[[2053,2270]],[[2274],[2277],[2278],[2279],[2280],[2282],[2271],[2284],[2286],[2272],[2273]],[[2283]],[[2287]],[[2288]],[[238,2127,3875]],[[2276,2127,3875]],[[604],[441]],[[23,2127,3875]],[[324,2127,3875]],[[181,2127,3875]],[[368,2127,3842]],[[553],[0]],[[2281,163,2127,3801]],[[848,2127,3853]],[[2285]],[[638],[374]],[[75,2127,3859]],[[189,2127,3875]],[[158,2127,3853]],[[2291],[0]],[[2289,2032,2166,636,3788,2169]],[[394,458,2165],[2292]],[[9,763,2293]],[[602],[335],[578]],[[537,496,2295]],[[130],[250]],[[2302],[0]],[[2032,594,2299,3786,2300,2301,383,3810,200,153,487,2296,3425]],[[2180]],[[2298]],[[28],[6]],[[242],[614],[133]],[[2304]],[[197],[415]],[[2303,3872]],[[2309],[0]],[[2312],[0]],[[2313],[0]],[[2032,170,2180,3796,383,490,3510,2305,2306,2307,147,3425]],[[383,79,2041,418]],[[2311],[0]],[[383,514]],[[156],[140,2310]],[[75,3859]],[[659,2180,2984]],[[2317,2315],[2317],[0]],[[394,458,523,718,710,3844,2315],[523,718,710,2180,3844,2315]],[[357,580,3861],[715,580,3861],[717,3861,230,45,3844],[716,580,3861]],[[148,2322]],[[2349]],[[2350]],[[2351]],[[2324],[2325],[2326],[2327],[2328],[2330],[2335],[2337],[2341],[2345],[2347],[2319],[2320],[2321]],[[3691],[0]],[[109,2323,3781]],[[170,2323,3797]],[[206,2323,3785]],[[422,2323,3783]],[[2057,236,3776,383,3810,2228]],[[2333],[0]],[[288,217,3795,2329]],[[2332,2331],[2332],[0]],[[2053,2334]],[[2334,2331]],[[2284],[2282]],[[503,2323,3800]],[[2339],[0]],[[2185,2338,2323,3813,2336]],[[574],[571]],[[471],[49]],[[2344],[0]],[[572,3793,2340]],[[2343,2342],[2343],[0]],[[2053,2334]],[[2334,2342]],[[594,2323,3787]],[[2348],[0]],[[636,2323,3701,2346]],[[471],[49]],[[659,2323,2984]],[[523,718,710,2323,3844]],[[605,572,3793,2152]],[[2355,2352],[2355],[0]],[[453,2354,2356,2352]],[[574],[571]],[[750,2356]],[[3810,590,3802]],[[574],[0]],[[597,2357,3810]],[[234,574,203,3863]],[[2363],[0]],[[48,3783,2360]],[[3404],[0]],[[753,2362,748]],[[2368],[0]],[[2380,2365],[2380],[0]],[[2364,133,2365,2377]],[[2543]],[[2367]],[[2636]],[[2371],[0]],[[2369]],[[2585],[0]],[[2378],[0]],[[2550],[0]],[[2499],[0]],[[3816,621,2555,2372],[3810,2370,2373,2372,2374,2375]],[[203,2376],[3816,203,2555,2372]],[[2379]],[[405,753,3832,748]],[[431],[295],[431],[232]],[[147,2384]],[[2578]],[[3404]],[[2382],[2383]],[[219,2389]],[[2498],[0]],[[66],[435,2390,2372,2386]],[[2636],[0]],[[3810,387,2388],[3830,2387]],[[2391],[3830,2394]],[[191],[367]],[[191],[367],[419],[268]],[[763],[769],[765],[768],[764]],[[2392],[2393,753,2424,748]],[[2408],[0]],[[232],[0]],[[248],[0]],[[3763],[0]],[[2429],[0]],[[242,2395,2396,2397,3810,2398,2407,2399]],[[2428]],[[2403],[0]],[[2401]],[[2428]],[[2406],[0]],[[2404]],[[2410,2402],[506,3704,2405],[2418]],[[295],[131],[223]],[[2412],[0]],[[2409,2416]],[[2414],[0]],[[753,2411,748]],[[2415,2413],[2415],[0]],[[3774,2413]],[[750,3774]],[[2417,2421]],[[626],[627]],[[2201],[753,2411,748,2201]],[[2424],[0]],[[2422,2420],[2422],[0]],[[753,2419,748,2420]],[[750,753,2419,748]],[[2427,2423],[2427],[0]],[[2425,2423]],[[3191],[128]],[[3191],[128]],[[750,2426]],[[17,3830,2168]],[[383,151,265,614,3704]],[[2438],[0]],[[284],[0]],[[2439],[0]],[[3708],[0]],[[2441],[0]],[[3710],[0]],[[3714],[0]],[[281,2440,2430,2431,237,3859,2432,248,574,3810,2398,2433,2434,2435,2436,2445]],[[295],[82]],[[458],[232]],[[112],[653]],[[484,230,45,3855]],[[2447],[0]],[[2450],[0]],[[2448],[0]],[[2442,2443,2444]],[[278],[484]],[[232,787,2446]],[[506,3704]],[[2452],[0]],[[753,2449,748]],[[2455,2451],[2455],[0]],[[2453,2451]],[[3773],[3383]],[[3773],[3383]],[[750,2454]],[[2458],[0]],[[458,2456,2397,3810,2398,2459]],[[295],[131]],[[2410],[506,3704],[2418]],[[2566],[0]],[[2465,2460],[2462]],[[753,2462,748],[2465,2506,2460],[2465,2566,2506]],[[2467],[0]],[[2470],[0]],[[2463,2468,2464]],[[2543]],[[2466]],[[2472,2374,2386],[2483,2374,2386]],[[2513]],[[2469]],[[2476,2471],[2476],[0]],[[2478,2471]],[[663]],[[608],[2473]],[[2634],[0]],[[2474,2475,2478]],[[2482,2477],[2482],[0]],[[2479,2477]],[[2484],[2483]],[[2484],[2483]],[[811,2475,2480]],[[2481]],[[753,2465,2460,748]],[[2493],[2485],[2486]],[[2558]],[[2560]],[[2562,2487],[2562],[0]],[[2506],[0]],[[2552],[0]],[[2547],[0]],[[2517],[0]],[[2495],[0]],[[497,2487,2578,2488,2489,2372,2490,2491,2492]],[[2519]],[[2494]],[[2483]],[[10],[143],[555],[223],[536],[531],[532],[534]],[[276,2501]],[[276,2504]],[[2503],[0]],[[2504,2500]],[[750],[381]],[[2502,2504]],[[3830],[2505]],[[754],[791],[788],[787]],[[248,2511]],[[3872],[3383]],[[3872],[3383]],[[2510,2509],[2510],[0]],[[750,2508]],[[396,3853,2433,2435,2436],[150,3853],[2507,2509]],[[2516],[0]],[[422,13,753,2512,748]],[[2515],[0]],[[750,787]],[[787,2514]],[[221,3191]],[[2520,2518],[2520],[0]],[[699,2521,2518]],[[750,2521]],[[3826,17,2522]],[[753,2528,748]],[[2533],[0]],[[2529],[0]],[[2530],[0]],[[3826],[0]],[[2531],[0]],[[405,45,3416,2374,2523],[2524,2550,2523],[2525,2374,2533],[2526,2527,2374,2523]],[[405,45,3416]],[[405,45,3416]],[[405,45,3416]],[[2539],[0]],[[2534,2535,2532]],[[484],[432],[683]],[[2536],[2537]],[[698,693],[3843,693],[754,693],[247,3191,3409,693],[101,487]],[[30,2538,15,2538]],[[2536],[698,682],[3843,682],[754,682],[247,3191,3409,682]],[[680,2540]],[[101,487],[217],[697],[373,690]],[[665],[0]],[[2544,2542],[2544],[0]],[[645,2541,2545,2542]],[[750,2545]],[[3830,2168,17,2496]],[[2548],[0]],[[217,45,3416,2546]],[[645,481],[2549]],[[645,99]],[[393,45,3416]],[[18],[134]],[[203,2553]],[[149],[2555]],[[2556,2554],[2556],[0]],[[2587,2554]],[[750,2587]],[[2559,2557],[2559],[0]],[[626,2561,2557]],[[750,2561]],[[574,3810]],[[487,753,2419,748]],[[2497],[535],[2563],[2564]],[[533]],[[325,763,3842]],[[2569,2565],[2569]],[[2565]],[[2571],[0]],[[2573],[0]],[[200,2574,2567,2568],[287,251,508,346]],[[668,3816]],[[2570]],[[2576]],[[2572]],[[614],[2575]],[[508]],[[669,670],[671]],[[2580,2577],[2580],[0]],[[2579,2577]],[[2582],[775]],[[750,2582]],[[2583],[0]],[[3778],[3191,2581]],[[2198,2584]],[[3830],[3853]],[[643,3191]],[[2593,2586],[2593],[0]],[[2590,2586]],[[3830]],[[2588],[732]],[[2605],[752,2589,2591,747]],[[2605,2586]],[[2594],[0]],[[2601,2587,2592],[2603,2587,2595],[2598,2605]],[[383,3191],[621,3834]],[[383,3191],[621,3834]],[[239],[0]],[[395],[0]],[[359,2596,261],[359,2599,2597,261]],[[272],[478]],[[2602],[0]],[[2600,261],[555]],[[239],[98]],[[2604,2597,261]],[[272],[478]],[[2608],[2609],[2612],[2616],[2606]],[[2618]],[[2640],[0]],[[3810,2398,2388,2607]],[[753,2610,748]],[[2608],[2609]],[[2614],[0]],[[2496,2388,2611],[2615]],[[3771]],[[2613]],[[726,2496,2388,2168]],[[753,2617,748]],[[2555],[2616]],[[701,753,3191,750,3853,2620,748,2388]],[[2621,2619],[2621],[0]],[[71,753,2625,2619,748]],[[750,2625]],[[2627],[0]],[[174],[0]],[[2630],[0]],[[3830,200,703],[3830,3592,2622,2623,704,3853,2624],[702,704,3853,2620]],[[3697]],[[2626]],[[2632],[0]],[[2631],[0]],[[2631,2628],[2632,2629]],[[2633,383,700]],[[2633,383,165]],[[165],[376],[128,3853]],[[143],[10]],[[2638],[0]],[[2635,3830]],[[763]],[[17],[2637]],[[2643,2639],[2643]],[[2639]],[[2648],[0]],[[2651],[0]],[[2644,2645,2641,753,2651,748],[620,2645,2641,753,2642,748]],[[198],[232]],[[265],[236]],[[2645],[0]],[[420,265],[609,2646]],[[200,2649]],[[261],[393,45],[217,45]],[[2652,2650],[2652],[0]],[[2653,2650]],[[750,2653]],[[3830],[420]],[[2658],[0]],[[295],[0]],[[2654,614,2655,2396,2555,506,3704,2372,2374,2375]],[[2543]],[[2657]],[[2664],[2675],[2683],[2691]],[[2670],[0]],[[647],[0]],[[2666],[0]],[[2667],[0]],[[543,592,2660],[77,2661,2662,2663]],[[373],[0]],[[15,2665,54]],[[2665,450]],[[29,2661]],[[2671,2669],[2671],[0]],[[2672,2669]],[[750,2672]],[[645,85,517],[2674]],[[649],[386]],[[435,2673]],[[489,3830],[480,2661,2681],[450,489,3830]],[[2677],[0]],[[15,2665,54]],[[2679],[0]],[[2665,450]],[[489],[0]],[[590,2680,3830],[2676,2678]],[[2685,2682],[2685],[0]],[[287,2684,2689,2682],[2686],[611,2688]],[[571],[574]],[[750,2689]],[[287,244,200,27]],[[244]],[[571],[574],[2687]],[[3810,2388,2690]],[[435,2431],[2655,649]],[[651,2701]],[[543],[29]],[[2694],[0]],[[261],[472]],[[2696],[0]],[[200,340]],[[2698],[0]],[[566,2695]],[[2700],[0]],[[384,407]],[[2692,2707,2693],[159,2707,2697],[417,2707],[77,2707,2699],[480,2707],[439,2702]],[[2705],[0]],[[2704],[0]],[[94,652]],[[2703]],[[2710],[0]],[[3855,2706]],[[2709],[0]],[[750,3841]],[[750,3855,2708]],[[2715,2711],[2715],[0]],[[428,2713,289,2714],[2733],[468,2724,2711],[2718],[2804],[2719],[2731],[2720]],[[32],[316]],[[590,3859],[28,3191]],[[750,2724]],[[2717],[0]],[[2323,3387]],[[468,658,2716]],[[2749]],[[2827]],[[2726],[0]],[[10],[0]],[[3423],[0]],[[316,2721],[2725],[514,2722,2723]],[[430,47]],[[2730]],[[3842]],[[3844]],[[2727],[2728]],[[590,2729]],[[281,2732,203,316]],[[112],[574,3810]],[[55,316,590,2735,2723]],[[2736,2734],[2736],[0]],[[2737,2734]],[[750,2737]],[[300,763,3861],[729,763,3861],[297,763,3861],[318,763,3861],[303,763,3861],[304,763,3841],[298,763,3841],[305,763,3841],[299,763,3841],[314,763,3841],[308,763,3861],[307,763,3861],[317,763,3861],[309,763,3861],[738,763,2740],[310,763,3861],[313,763,3861],[315,763,3841],[311,763,3859],[312,763,3861],[712,763,3861],[713,763,3841],[319,763,3841],[233,763,2743],[735,763,3853],[736,763,3841],[296,763,3841],[737,763,2738],[739,763,3841],[742,763,2739],[2741]],[[3755],[376]],[[743],[383],[744]],[[3861],[376]],[[301,763,3861],[302,763,3843],[447,763,3861],[448,763,3841]],[[2746],[0]],[[753,2742,748]],[[2745,2744],[2745],[0]],[[750,3841]],[[3841,2744]],[[2750,2747],[2750],[0]],[[2752],[0]],[[55,459,522,590,2754,2723],[55,459,190,2788,2747,2748]],[[750,2788]],[[3423]],[[2751]],[[2755,2753],[2755],[0]],[[2756,2753]],[[750,2756]],[[2757,763,3861],[2758,763,3861],[2759,763,3861],[2760,763,3861],[2761,763,3841],[2762,763,3861],[2763,763,3843],[2764,763,3841],[2765,763,3841],[2766,763,3841],[2767,763,3841],[817,763,3841],[2768,763,3841],[2769,763,3853],[2770,763,3841],[2771,763,3841],[2772,763,3861],[2773,763,3861],[2774,763,3861],[2775,763,3859],[2776,763,3861],[2777,763,3861],[2778,763,3861],[2779,763,3841],[2780,763,3861],[2781,763,2740],[2782,763,3861],[2783,763,3841],[729,763,3861],[233,763,2743],[841,763,3841],[737,763,2738],[739,763,3841],[742,763,2739],[842,763,3841],[447,763,3861],[448,763,3841]],[[814],[297]],[[820],[300]],[[838],[318]],[[823],[303]],[[824],[304]],[[821],[301]],[[822],[302]],[[813],[296]],[[819],[319]],[[816],[298]],[[826],[305]],[[818],[299]],[[815],[735]],[[839],[736]],[[827],[314]],[[828],[308]],[[829],[307]],[[830],[309]],[[832],[311]],[[833],[312]],[[834],[313]],[[831],[310]],[[835],[315]],[[837],[317]],[[836],[738]],[[825],[712]],[[840],[713]],[[2790],[0]],[[2793],[0]],[[2796],[0]],[[2800],[0]],[[460,763,753,2784,748],[461,763,753,2784,748],[462,763,753,2785,748],[463,763,753,2785,748],[464,763,753,2786,748],[465,763,753,2786,748],[466,763,753,2787,748]],[[2791,2789],[2791],[0]],[[3781,2789]],[[750,3781]],[[2794,2792],[2794],[0]],[[3804,2792]],[[750,3804]],[[2797,2795],[2797],[0]],[[2798,2795]],[[750,2798]],[[3861]],[[2801,2799],[2801],[0]],[[3699,2799]],[[750,3699]],[[2824],[0]],[[2805],[0]],[[543,514,2802,2803,2813,2723],[552,514,2802,2723]],[[613,2807]],[[2812,2806],[2812],[0]],[[2811,2806]],[[530],[528]],[[2808,763,3855]],[[529]],[[2741],[2809],[2810]],[[750,2741]],[[2822],[0]],[[2815],[0]],[[618,763,3855]],[[2817],[0]],[[406,763,3855]],[[2819],[0]],[[129,763,3855]],[[2821],[0]],[[409,763,3855]],[[2814,2816,2818,2820]],[[2825,2823],[2825],[0]],[[2826,2823]],[[750,2826]],[[449],[538]],[[2828,210]],[[543],[552]],[[417,3830,203,2830],[2833],[2831,417,3830]],[[3859],[3383]],[[123],[148]],[[2834],[0]],[[173,3830,2832]],[[621,2836]],[[2837,2835],[2837],[0]],[[3383,2835]],[[750,3383]],[[677,2844]],[[2840],[0]],[[200,459]],[[2846],[0]],[[244,203,3758,749,3841,230,45,3853,2841]],[[3877],[0]],[[284,112,139,2843,3853],[676,2839],[2842]],[[2847],[0]],[[2847],[112,139,2843,3853,2845]],[[467,2665,539]],[[2849],[2863],[2897],[2900],[2929],[2933],[2850]],[[2852]],[[2979]],[[2854],[0]],[[11,618,2851,2855]],[[3691]],[[2853]],[[2858],[2861,2866]],[[2862],[3758]],[[10],[369],[2984]],[[2856,128,659,2857]],[[3724]],[[3721]],[[2859],[2860]],[[618,3876]],[[97,618,2865,3721,2875,2866]],[[3692]],[[2864],[0]],[[2874],[0]],[[75],[812]],[[2867,3855]],[[2870],[0]],[[2868]],[[2879],[0]],[[2883],[0]],[[2885,2873],[2885],[0]],[[2871,2872,2873,2869]],[[2878],[0]],[[2877],[0]],[[128,659,2984]],[[2876]],[[467,2881]],[[539],[650],[369]],[[2973],[2880]],[[2884,2882],[2884]],[[645,2882]],[[322,3841],[327,3841],[321,3841],[328,3841]],[[2,2886],[406,2894],[740,2895],[741,3842]],[[287],[611]],[[2888],[0]],[[247,3842,122],[365],[128]],[[3842],[128]],[[3842,122],[128]],[[2892],[0]],[[128],[719]],[[467,101,2891]],[[177,2887],[705,2889],[706,247,2890],[2893]],[[3842],[698]],[[2899],[0]],[[148,618,2896,3718]],[[3691]],[[2898]],[[215,2912]],[[2902],[0]],[[645,660,391]],[[2952,590,3718,2901]],[[421],[0]],[[2952],[10,2904]],[[2907],[0]],[[645,215,391]],[[2950],[0]],[[2926],[0]],[[2916],[0]],[[2925],[0]],[[2903],[2905,383,2908,2968,590,2913,2909,2910,2911],[427,383,3758,590,2913,2906]],[[2914],[2915]],[[3721]],[[3718]],[[2918],[2919]],[[2977,2917],[2977]],[[645,2917]],[[645,215,391]],[[663,2984]],[[645,659,2923]],[[2920],[0]],[[2984],[10,2922],[369],[128]],[[2921],[0]],[[17,618,2924]],[[2927]],[[2879]],[[2930,2928],[2930],[0]],[[453,618,3758,590,3758,2928]],[[750,3758,590,3758]],[[2935],[0]],[[2944],[0]],[[477,2931,2942,2932]],[[3691]],[[2934]],[[2952,203,3718]],[[2945],[0]],[[2939],[0]],[[2937,203,3718]],[[383,2908,2968,2938]],[[2940],[750,215,391,203,3718]],[[2936],[2952,2945,203,3718],[10,2904,2941],[427,383,3758,203,3718]],[[232,610,618]],[[2943]],[[2946],[2949]],[[383,2908,2968]],[[2948],[0]],[[383,2908,2968]],[[2947]],[[574],[206],[422]],[[2953,2951],[2953],[0]],[[2956,2951]],[[750,2956]],[[2965],[0]],[[483],[0]],[[2959],[2960,2168],[2962],[2963],[215,391],[509,110],[97,2954],[287,571],[459,2966],[509,636],[11,2955]],[[792],[746,3872]],[[3874,2957],[3874,2168]],[[2958]],[[497],[242],[614],[443]],[[97],[148]],[[2961,659]],[[133],[616],[236],[148],[173],[451],[510],[423],[188],[427],[565],[170],[594]],[[483],[572],[618],[636]],[[577,571],[2964]],[[65],[514]],[[2969],[0]],[[775,2967],[3781,751,2971],[3781],[3810]],[[751,775]],[[3810]],[[775],[2970]],[[2975,2972],[2975],[0]],[[2976,2972]],[[15],[0]],[[2974,2976]],[[63,3855],[259,3855],[559,3855]],[[215,391],[322,3841],[327,3841],[321,3841],[328,3841]],[[2982],[0]],[[506,659,2984],[506,659,2980],[506,128,659,2981,590,2984],[506,659,10,2978]],[[369],[128]],[[2984],[369],[10]],[[663,2984]],[[2985,2983],[2985],[0]],[[2987,2983]],[[750,2987]],[[2988],[0]],[[3874,2986]],[[746,3872],[792]],[[2994],[0]],[[2997],[0]],[[14,2077,2992,3813,2989],[62,2995,3813,2078],[61,2996,3813,2990],[388,2077,2998,3813],[455,2077,2999,3813,2079]],[[574],[571]],[[3002]],[[2993]],[[574],[571]],[[574],[571]],[[431],[180]],[[574],[571]],[[574],[571]],[[3003],[0]],[[3004],[0]],[[614,674,383,3832,3000,3001],[148,674,383,3832]],[[645,787,675]],[[621,112,3853]],[[200,615],[3006]],[[431],[184],[333],[180],[56]],[[431],[180],[619]],[[3011],[0]],[[3012,3009],[3012],[0]],[[245,410,3830,520,3853],[245,664,3863,3008],[607,410,3823],[607,664,3824,3009]],[[506,3018]],[[750,3824]],[[214],[658]],[[3013],[0]],[[3014,3387,3877,3016]],[[383],[3191]],[[3019,3017],[3019],[0]],[[3015,3017]],[[750,3015]],[[506,3022]],[[3023],[0]],[[592,3034],[406,3021,3877,3028],[3031],[3045,3043],[3878,3051]],[[200,3758]],[[382,753,3855,748]],[[406,753,3855,748]],[[3753],[0]],[[3751],[0]],[[3855,3026,3027],[3855,3026,3027],[3024],[3025]],[[3030],[0]],[[200,3758]],[[406,3029,590,734,3026,3027]],[[3035],[0]],[[3036],[0]],[[3037,3032],[3039,3033]],[[750,3039]],[[750,3037]],[[435,3038]],[[649],[386]],[[258,274,3041]],[[76],[601]],[[456,435],[435,3040],[500]],[[3044,3042],[3044],[0]],[[3042]],[[750,3048]],[[3387,3877,3053],[3708],[3383,3877,3191],[3050,3877,3053],[356,3047]],[[128]],[[3877,3191],[3618,2096],[3046]],[[3878,3387,3877,3053],[3045]],[[3880],[0]],[[745,3049,3387]],[[3052,3043],[592,3034]],[[3387,3877,3053]],[[3191],[3054],[3056]],[[128],[383],[10],[32]],[[487],[710]],[[3055]],[[509,3091]],[[22]],[[3801],[10]],[[547],[354],[289]],[[203],[251]],[[32],[316]],[[225],[547,3094,2723]],[[33],[446]],[[3066],[0]],[[251,3855]],[[3068],[0]],[[203,3843]],[[180]],[[3071],[0]],[[3069]],[[236],[235],[263]],[[639],[166]],[[3075,3074],[3075],[0]],[[750,3099]],[[3077],[0]],[[3099,3074]],[[3079],[0]],[[200,430,787]],[[547],[631]],[[93]],[[3083],[0]],[[200,3758]],[[618,3758]],[[109,2180,3781],[170,3797],[206,3785],[422,3783],[574,3810],[594,3787],[636,3790],[3084]],[[3760],[0]],[[3092],[0]],[[3098],[0]],[[204],[0]],[[3878],[0]],[[3058],[110,3086],[3087,571,3088,3086],[3089,593,3088,3086],[169,3088,3086],[574,547,3088,3086],[387,571,3088,3086],[408],[163,3059,3060],[3087,71,3061,3810,3088,3086],[3062,289],[514,3063],[3064,169,3065,3067,2386,2723],[3070,3072,3097,3810,3088,2372],[2281,162],[95,753,775,748,3073],[639,2386],[166,2386],[426],[425,3076,3078,2386],[3090,3080,3086],[3089,424],[3406,3086],[70,3086],[3081],[421],[216,200,3758,621,3718],[216,3082],[316,547],[97,3085],[422,547,3086],[206,547,3086],[422,68,3783],[206,68,3785]],[[204],[3093]],[[180,3089]],[[3096],[0]],[[370],[0]],[[3095]],[[203],[251]],[[3097,3830]],[[40,255],[91,568],[400,185],[3100]],[[10],[96],[256],[334],[522],[567]],[[3107],[0]],[[33,3859],[47,236,3109,251,3103],[196,2077,3106],[266,3101,3191],[281,236,248,47,3136],[3108]],[[3830],[128]],[[3105,3104],[3105],[0]],[[750,3123]],[[3129],[3123,3104]],[[84],[430]],[[510]],[[3111],[3115]],[[3112,3110],[3112],[0]],[[3114,3110]],[[750,3114]],[[3117],[0]],[[3810,3113]],[[3810,405,753,2139,748,3113]],[[3120],[0]],[[2645,753,3116,748]],[[3830],[420]],[[3121,3119],[3121],[0]],[[3118,3119]],[[750,3118]],[[3127],[0]],[[3124],[3122,289],[445,289,2723],[3125],[3126]],[[136],[225],[421],[547],[617]],[[430,47]],[[389]],[[32],[163],[165],[208],[515]],[[3132],[0]],[[3130,3128]],[[571],[574]],[[3133],[0]],[[645,435,287],[3813,3131]],[[3134],[645,435,287]],[[200,179]],[[3137],[0]],[[3810,3144,3113,3135],[3139]],[[232,270]],[[3140,3138],[3140],[0]],[[3142,3138]],[[750,3142]],[[3143],[0]],[[3810,3113,3141]],[[232,270]],[[405,753,2139,748]],[[3149],[3160],[3162],[3167]],[[3152],[0]],[[3157],[0]],[[3158],[0]],[[97,709,217,3830,599,2843,3150,3146,3147,3148]],[[618],[710]],[[3153,3151],[3153],[0]],[[711,2843,3155,3151]],[[2053,3155]],[[3156],[0]],[[787,3154]],[[773,787]],[[708,2843,787]],[[156],[140]],[[198],[0]],[[11,709,217,3825,3146,3147,3148,3159]],[[3163],[0]],[[506,709,217,3830,3161]],[[200,3165]],[[3166,3164],[3166],[0]],[[3842,3164]],[[2053,3842]],[[148,709,217,3825,3159]],[[3175],[3171],[3187],[3188],[3169]],[[3189]],[[3173],[0]],[[3172,3810,3170]],[[178],[135],[134]],[[3855],[3773]],[[3182],[0]],[[3176,3174,3183]],[[178],[135],[134]],[[180]],[[404]],[[201,763,3872]],[[14,201,763,3872]],[[14]],[[3177],[3178],[3179],[3180],[3181]],[[2461],[3185],[3186]],[[2366],[2400],[2457],[2656]],[[3184]],[[200,84,3842]],[[222,3872]],[[620,3830]],[[714]],[[3198,3190],[3198],[0]],[[3193,3190]],[[3196],[0]],[[3202,3192],[3197]],[[596],[183],[610]],[[3407],[0]],[[257,3195,3194]],[[371,3191]],[[3199,3191],[654,3191],[3200,3191]],[[15],[770]],[[394],[772]],[[3203,3201],[3203],[0]],[[3207,3201]],[[257,3195,376],[3205,3204,2496],[3205,3207]],[[10],[16]],[[763],[777],[764],[765],[768],[769],[776]],[[3210],[0]],[[3216,3206]],[[668],[0]],[[733,3208,3414]],[[3195,3212],[3209],[521,275,3216]],[[3214],[0]],[[251,3213],[30,3216,15,3207],[275,3223,3211],[444,3216]],[[2496],[753,3404,748]],[[168,3223]],[[3217,3215],[3217],[0]],[[3223,3215]],[[760,3216],[3218,3216],[3219,3216],[3220,247,3191,3409],[3221,3216],[757,3216],[759,3216]],[[775],[762],[774],[145],[349]],[[778],[773]],[[778],[773]],[[779],[780]],[[3224,3222],[3224],[0]],[[3226,3222]],[[761,3226]],[[3227],[0]],[[3236,3225]],[[69,3872]],[[3237],[0]],[[487],[0]],[[3320],[0]],[[3246],[0]],[[3191],[0]],[[3245,3233],[3245]],[[3393],[0]],[[3248],[0]],[[3847],[3265],[3382,3228],[754],[3238],[3239],[3240,3223],[3408,3223],[3229,753,3404,748],[2623,2496],[752,3830,3191,747],[320,3315,7,753,3216,3230,748],[32,3223],[3244],[52,753,3191,17,3398,3231,748],[51,3232,3233,3234,159],[94,753,3191,750,3398,748],[94,753,3191,621,3618,748],[128,753,3837,748],[626,753,3837,748],[247,3191,3409,778,3191],[3377],[3329],[3773,3235]],[[3877,3191]],[[3294]],[[3298]],[[778],[773],[758]],[[247],[0]],[[3617],[0]],[[52,753,3191,21,586,843,3241,3853,17,113,3242,748]],[[3243]],[[3391,3392]],[[3247]],[[731]],[[3249],[3250]],[[766,3853]],[[767,3853]],[[143],[0]],[[3267],[0]],[[3270],[0]],[[3273],[0]],[[3276],[0]],[[3278],[0]],[[3280],[0]],[[3282],[0]],[[3284],[0]],[[3286],[0]],[[3288],[0]],[[3290],[0]],[[3291],[0]],[[3293],[0]],[[26,753,3251,3314,748,3252],[3268,753,3314,748,3253],[3271],[95,753,2722,775,748,3254],[95,753,3274,748,3255],[345,753,3251,3314,748,3256],[326,753,3251,3314,748,3257],[551,753,3314,748,3258],[632,753,3314,748,3259],[548,753,3314,748,3260],[635,753,3314,748,3261],[564,753,3251,3314,748,3262],[218,753,3251,3404,2374,3263,748,3264]],[[3304]],[[3266]],[[35],[36],[38]],[[3304]],[[3269]],[[3313]],[[3304]],[[3272]],[[2722,775],[3314],[143,3404]],[[3304]],[[3275]],[[3304]],[[3277]],[[3304]],[[3279]],[[3304]],[[3281]],[[3304]],[[3283]],[[3304]],[[3285]],[[3304]],[[3287]],[[3304]],[[3289]],[[499,3855]],[[3304]],[[3292]],[[672,753,3404,748]],[[3307],[0]],[[3310],[0]],[[3303],[0]],[[3299,3876,3304],[688,3414,3304],[3300,753,3191,3295,748,3296,3304],[3301,3413,3296,3304],[687,753,3191,750,3223,748,3297,3296,3304]],[[696],[694],[679],[678],[692]],[[686],[684]],[[681],[685]],[[191],[268]],[[203,3302]],[[691,3305]],[[3826],[2522]],[[3309],[0]],[[750,3308,3306]],[[3843],[754],[3830],[3383]],[[750,3191]],[[3311,689]],[[695],[232]],[[3304],[0]],[[667,753,3314,748,3312],[666,753,3314,750,3314,748,3312]],[[2722,3191]],[[3317],[753,3317,748]],[[3318,3316],[3318],[0]],[[3837,3316]],[[750,3837]],[[3321],[0]],[[251,41,346],[251,359,267,346,3319],[645,430,176]],[[645,430,176]],[[3330],[0]],[[3876],[0]],[[3331,3324],[3331]],[[3335],[0]],[[3353],[0]],[[3340],[0]],[[3343],[0]],[[60,753,3404,3322,748],[105,3323],[116,3413],[122,3413],[229,3413],[242,753,3191,750,3191,750,3191,750,3191,748],[247,753,3191,3324,748],[3334],[272,753,3191,750,3191,748],[343,3413],[350,3413],[478,753,3191,750,3191,748],[495,3413],[586,3413],[583,753,3191,3325,748],[3366],[618,3876],[626,3413],[656,3413],[3336,753,3191,750,3337,748],[100,3323],[108,3326],[3338,753,3191,750,247,3191,3409,748],[182,753,3409,203,3191,748],[213,753,3365,750,3191,748],[372,3326],[414,753,3216,251,3191,748],[3370],[569,3326],[3339,753,3411,750,3191,750,3191,748],[622,3323],[624,3326],[623,3326],[19,3413],[58,3413],[67,3412],[70,3413],[109,3876],[231,753,3191,750,3191,750,3191,748],[201,753,3191,750,3191,3327,748],[337,3413],[349,753,3191,750,3191,748],[3341],[3342],[429,3413],[457,753,3191,750,3191,748],[458,753,3191,750,3191,750,3191,748],[476,3413],[485,3876],[597,753,3191,750,3191,748],[640,753,3191,3328,748],[641,753,3191,17,60,748],[641,753,3191,3349,748],[3350]],[[621,3618]],[[750,3191]],[[3333],[0]],[[851,3398]],[[850,753,3223,750,3859,3332,2624,748]],[[750,3191]],[[5],[558]],[[3191],[247,3191,3409]],[[114],[115]],[[584],[585]],[[750,3191]],[[382,753,3859,748]],[[406,3413]],[[750,3191]],[[3345],[0]],[[17,60,3616]],[[3356]],[[3348],[0]],[[3346]],[[17,32,3616],[3344,3347],[750,3841,750,3841,750,3841]],[[3351],[211,753,2362,748],[279,3412],[351,3412],[352,3412],[353,3412],[411,753,3191,750,3191,748],[412,3412]],[[90,753,3191,750,3191,748]],[[3354],[0]],[[753,3352,748]],[[3355]],[[787]],[[274,3359]],[[3358,3357],[3358],[0]],[[750,3361]],[[3842,773,3842],[3361,3357]],[[3364],[0]],[[3842,3360]],[[18],[134]],[[476],[0]],[[3362,3363],[476]],[[116],[586],[113],[583]],[[595,753,3369,748]],[[3368],[0]],[[203,3191]],[[3191,3367],[269,3232,203,3191],[591,3232,203,3191],[43,3232,203,3191]],[[563,753,3191,3375,748]],[[3372],[0]],[[750,3191]],[[3374],[0]],[[200,3191]],[[750,3191,3371],[203,3191,3373]],[[3379],[0]],[[3827,753,3376,748],[3835,753,2362,748]],[[3380,3378],[3380],[0]],[[3381,3378]],[[750,3381]],[[3191,2581]],[[3383],[3386]],[[746,3872],[792]],[[3879],[0]],[[3840],[0]],[[745,3384,3872,3385]],[[3390],[128,3840]],[[3830,3385]],[[3873,3385]],[[3388],[3389]],[[642,3191]],[[582,3191]],[[154,3191]],[[3606],[0]],[[3612],[0]],[[249],[0]],[[3869],[0]],[[32,3394],[60,3394,3395],[3604,3394],[512,3396],[612,3396],[116],[586,3242],[113,3242],[126,3397],[656],[3399],[3400],[3402]],[[262]],[[3605]],[[3870],[0]],[[195,3401]],[[3405,3403],[3405],[0]],[[3191,3403]],[[750,3191]],[[60,506],[58]],[[371],[800]],[[771],[800]],[[3411],[3410]],[[494],[341],[342],[226],[228],[227],[119],[121],[120],[118],[655]],[[337],[495],[343],[229],[122],[640],[350],[429],[656]],[[753,3404,748]],[[753,3191,748]],[[753,3223,748]],[[3417,3415],[3417],[0]],[[3418,3415]],[[750,3418]],[[3191,2123]],[[3421,3419],[3421],[0]],[[3422,3419]],[[750,3422]],[[3191]],[[3424]],[[200,57,3861]],[[2004],[3426],[3427],[3437],[3441],[3442],[3447],[3448],[3470],[3469],[3502],[3505],[3503]],[[475,3191]],[[231,3429,159,231]],[[3430],[0]],[[3191,3431,3428]],[[155,3429],[154,3433]],[[582,3433]],[[3434,3432],[3434]],[[3432]],[[3425,755]],[[3438,3435],[3438]],[[3439],[0]],[[51,3232,3435,3436,159,51]],[[3391,3431]],[[154,3433]],[[3820],[0]],[[3443,3446,3440]],[[3446]],[[3819,749]],[[3453],[0]],[[3433],[0]],[[29,3444,3445,159]],[[3443,3448,3440]],[[3449],[3450],[3451]],[[294,3433,159,294]],[[644,3191,147,3433,159,644]],[[457,3433,613,3191,159,457]],[[3454,3452],[3454]],[[3452]],[[3455,755]],[[3457],[3459],[3464],[3468]],[[3458],[0]],[[127,3832,3592,2096,3456]],[[128,3191]],[[127,3830,83,200,3460]],[[3841],[3462]],[[627],[0]],[[526,3461,3859]],[[3466,3463],[3466],[0]],[[127,3465,219,200,3467,3463,3425]],[[92],[175],[605]],[[750,3467]],[[3460],[3830],[527],[3407,202],[525]],[[127,3830,106,200,2461]],[[260,3820]],[[271,3820]],[[3474],[0]],[[207,3471,138,3479]],[[540]],[[101],[3473]],[[3476,3475],[3476],[0]],[[750,3481]],[[3478,3477],[3478],[0]],[[750,3484]],[[3481,3475],[83,3480,3484,3477]],[[3847],[3382],[3837]],[[3482,763,3483]],[[3382],[3830]],[[377],[485]],[[3485,763,3486]],[[3382],[3830]],[[3487],[473]],[[64],[557],[87],[89],[88],[53],[492],[576],[73],[107],[336],[355]],[[3493],[0]],[[511,3490,3488]],[[3830],[3462]],[[3492,3491],[3492],[0]],[[750,3501]],[[506,3501,3491]],[[3497],[0]],[[3500],[0]],[[469,3494,3495]],[[3830],[3462]],[[3499,3498],[3499],[0]],[[750,3501]],[[506,3501,3498]],[[3487,763,3480]],[[387,3830]],[[66,3830]],[[3507],[0]],[[186,3504,3830,248,3832]],[[367],[0]],[[3506,203]],[[3511],[0]],[[3512],[0]],[[21,3191],[171,3191,3409,3508,3509]],[[542,3191]],[[160,3191]],[[3765,3528,2098]],[[3515],[3556]],[[3516]],[[62,3413]],[[2041,730]],[[3527],[0]],[[3520,2232,3574,2235],[205,2646,2239,3574,2236],[523,2646,2239,3574,2237],[3518,3525]],[[265],[236]],[[420,265],[609,2646]],[[3517]],[[3524],[0]],[[3522]],[[3521,2232,3574,2235],[199,265,2239,3567,3556],[3516,3523]],[[3830],[0]],[[86,3526]],[[3592,3539]],[[3530],[0]],[[209,12]],[[3532],[0]],[[637],[554]],[[3542,3533],[3542],[0]],[[3533]],[[3552,3535],[3552],[0]],[[3535]],[[3534],[3536]],[[2096,3529,17,3413,3531,3537]],[[3538],[3533]],[[420],[0]],[[265],[0]],[[2041,3867],[3543],[128,3545],[3546],[383,614,372,3326],[24],[501,128,627],[3540,265],[609,3541],[75,3859],[3697],[74,3550],[553,3551],[3547],[3548],[3549]],[[371,720]],[[3413]],[[3849],[372,3326],[3544]],[[3582]],[[707,3844]],[[3518,3516]],[[3517]],[[192],[152],[128]],[[142],[334],[128]],[[609,3541],[75,3855],[3195,376],[3540,265]],[[3834],[0]],[[3558],[0]],[[3563],[0]],[[443,3810,3553,3554,3555]],[[204],[402],[513]],[[320,3557]],[[3560],[0]],[[383,133,3564]],[[3562],[0]],[[383,614,3564]],[[383,614,3564,3559],[383,133,3564,3561]],[[3565],[506,3867],[373,3],[506,128]],[[471],[49]],[[3568,3566],[3568],[0]],[[753,3569,3566,748]],[[750,3569]],[[3830,3394,2123]],[[3572,3570],[3572],[0]],[[753,3573,3570,748]],[[750,3573]],[[3569],[3413,2123]],[[3575],[3576]],[[3571]],[[3567]],[[3578]],[[44],[488],[220]],[[3580],[3583]],[[264,2127,3841],[75,3859],[3581]],[[3582]],[[662],[661]],[[3584,3577]],[[621],[599]],[[3580],[645,401,3830]],[[3580]],[[3592,-1]],[[3609],[0]],[[3871],[0]],[[32],[0]],[[3601],[0]],[[3593,3394,3588],[3595,3589,3588],[3596,3397,3588],[37,3394],[3597],[3598,3606,3395],[60,3394,3395],[32,3394],[3599,3606,3590],[3604,3394,3590],[628,3606],[656,3394,3588],[116],[586,3242],[583,3242],[113,3242],[587],[39,3394],[3600],[293,628],[293,3591,3395],[589,3395],[580,3394,3395],[332,3395],[291,3395],[164,3851,3395],[506,3851,3395],[501],[3602],[3603]],[[249],[588],[516],[331],[31]],[[416],[0]],[[437],[146,3594]],[[195],[126],[378],[192]],[[42],[41]],[[60,633],[629]],[[358,629],[379],[361,629],[358,60,633],[361,633]],[[330],[290]],[[60,633],[629]],[[262]],[[212],[211],[411],[352],[279],[351],[412],[353]],[[361],[358,60]],[[437],[146,3594]],[[753,3607,748]],[[3844],[783]],[[3610,3608],[3610]],[[3608]],[[512],[612],[657]],[[3613],[0]],[[3614],[3615],[46],[3406,3618,3590],[32,3611]],[[3406,3618]],[[19,3590],[32,19]],[[606,3590],[32,606]],[[753,3842,748]],[[753,787,748]],[[3872],[32],[3619]],[[128]],[[3872],[3621],[3622]],[[128]],[[32]],[[3625,3623],[3625],[0]],[[3629,3623]],[[2053,3629]],[[3629,3626],[3629]],[[3626]],[[3813],[0]],[[163,2127,3801],[3631],[323,2127,3843],[344,2127,3843],[25,2127,3841],[406,2127,3853],[75,2127,3853],[3632],[3633],[24,2127,3843],[399,2127,3645],[3634,2127,3645],[3635,2127,3841],[132,2127,3841],[486,2127,3636],[608,2127,753,3628,748],[3649],[3647],[243,2127,3637],[112,139,2127,3855],[236,139,2127,3855],[572,3639,3830],[553,3640],[84,2127,3855],[264,2127,3841],[3641],[3642],[3643],[3644]],[[376],[3872]],[[721,2843,3630]],[[81,2127,3855]],[[158,2127,3855]],[[544],[545],[546]],[[61],[575]],[[128],[152],[192],[80],[442],[78]],[[373],[191],[268]],[[2127]],[[3638],[0]],[[142],[334]],[[543,592]],[[848,2127,3855]],[[849,2127,3855]],[[2277]],[[3841],[128]],[[128],[0]],[[3646,69,2127,3620]],[[3646,158,2127,3853]],[[3646,3406,2127,3618]],[[3654],[0]],[[3662],[0]],[[3668],[0]],[[405,45,3658,3650,3651,3652]],[[404,3842]],[[277],[0]],[[3665],[0]],[[3832],[0]],[[3655,265,3656,753,3657,748],[3655,220,753,3216,748],[3659,3660]],[[432],[280]],[[753,3216,748],[71,753,3657,748]],[[3664],[0]],[[561,45,3655,3663,3661]],[[220,753,3216,748],[265,3656,3834]],[[560,3842]],[[3666]],[[9,763,3842]],[[3669,3667],[3669],[0]],[[753,3673,3667,748]],[[750,3673]],[[3675],[0]],[[3682,3671],[3682],[0]],[[3678],[0]],[[405,3830,3670,3671,3672]],[[3687],[329]],[[626,273,581,3674],[626,251,3680]],[[3677,3676],[3677],[0]],[[750,3685]],[[753,3685,3676,748]],[[3681,3679],[3681],[0]],[[3687],[753,3687,3679,748]],[[750,3687]],[[572,2127,3830],[2281,163,2127,3801],[368,2127,3842],[3683,2127,3842],[3684,139,2127,3859],[75,2127,3859]],[[323],[344]],[[112],[236]],[[561,3872,3671]],[[3688,3686],[3688],[0]],[[753,3689,3686,748]],[[750,3689]],[[3216],[329]],[[130,763,3758]],[[231,174]],[[231,3407,174]],[[3695],[0]],[[3693,3696]],[[251],[397],[240]],[[3818,3698]],[[69,3620]],[[3592,2096]],[[753,3781,750,3781,748]],[[3702,3700],[3702],[0]],[[3790,3700]],[[750,3790]],[[3705,3703],[3705],[0]],[[3706,3703]],[[750,3706]],[[3773,3877,3707]],[[3191],[128]],[[3406,3618]],[[3712,3709],[3712]],[[71,3709]],[[392],[0]],[[579,45,3855],[3711,157,45,3855],[167,45,3855]],[[3715,3713],[3715]],[[278,3713]],[[3716,45,3855]],[[579],[541]],[[3719,3717],[3719],[0]],[[3758,3717]],[[750,3758]],[[3722,3720],[3722],[0]],[[3727,3720]],[[750,3727]],[[3725,3723],[3725],[0]],[[3739,3723]],[[750,3739]],[[3738],[0]],[[3758,3726]],[[406]],[[3730],[0]],[[3728]],[[45,3855]],[[3733],[0]],[[17,3856],[3731]],[[3735],[0]],[[645,3872]],[[3734,45,734,406]],[[45,3729,3855],[645,3872,3732],[3736]],[[230,3737]],[[3740,3750]],[[2862],[3758]],[[3742],[0]],[[645,3872]],[[734,406]],[[3743],[3855]],[[3746],[0]],[[17,3856,3027]],[[3748],[0]],[[3741,45,3744,3026,3027],[645,3872,3745]],[[3752]],[[230,3747],[3749],[0]],[[727,101,406]],[[141,728,406]],[[458,3855]],[[3757],[0]],[[3872,3754]],[[3872],[0]],[[746,3756],[792]],[[3755],[105,3323]],[[275,3853]],[[3759],[2585]],[[385],[380]],[[284],[375]],[[3764]],[[405,3834]],[[3767],[3768]],[[3840],[3835,3385]],[[3766]],[[3830]],[[3830]],[[3772,3770],[3772],[0]],[[753,3769,3770,748]],[[750,3769]],[[3765]],[[3773],[3778]],[[3830]],[[3765]],[[3779],[0]],[[3830,751,3777,775]],[[3830,751]],[[3830]],[[3830]],[[3835]],[[3835]],[[3835]],[[3835]],[[3835]],[[3835]],[[3835],[3789]],[[3840]],[[3835],[3791]],[[3840]],[[3830]],[[3830]],[[3830]],[[3830]],[[3835]],[[3835]],[[3830]],[[3872]],[[3872]],[[3872]],[[3835],[3803]],[[3840]],[[3781,3840]],[[3809],[0]],[[3830,3805]],[[3808],[0]],[[751,775]],[[751,775],[3840,3807]],[[3835],[3811]],[[3840]],[[3814,3812],[3814],[0]],[[3810,3812]],[[750,3810]],[[3817,3815],[3817],[0]],[[3806,3815]],[[750,3806]],[[3830]],[[3827],[3889]],[[3819]],[[3827],[3898]],[[3821]],[[3830]],[[3853]],[[3830]],[[3830]],[[3828],[3829]],[[793],[781]],[[784]],[[3827],[3881]],[[3833,3831],[3833],[0]],[[3830,3831]],[[750,3830]],[[753,3832,748]],[[3830,3385]],[[3838],[0]],[[3830,3836],[3839]],[[3840,3385]],[[3840,3840]],[[751,3830]],[[787],[786],[788],[791],[783],[785]],[[787],[786],[788],[791]],[[787],[788],[791],[783],[785]],[[787],[3845],[791],[788]],[[786]],[[794],[0]],[[3859],[3865],[3868],[3867],[3866],[3846,3848]],[[786],[782]],[[3847],[778,3841],[773,3841]],[[3852,3850],[3852],[0]],[[753,3855,3850,748]],[[750,3855]],[[790],[3854]],[[784]],[[3853],[786],[782]],[[3853],[3857]],[[786]],[[3853,3858],[3853],[0]],[[3860,3858]],[[3846,3853],[789]],[[3853]],[[3864,3862],[3864],[0]],[[3853,3862]],[[750,3853]],[[787],[788],[791],[783],[785]],[[596],[183]],[[376],[801]],[[116,3853],[586,3853],[583,3853]],[[3606],[3871]],[[3606]],[[753,787,750,787,748]],[[3830],[3853]],[[3827],[3902]],[[3821],[3853]],[[3844],[3827]],[[753,748]],[[763],[756]],[[658],[673],[214],[284],[502]],[[214,751],[284,751],[502,751]],[[658,751],[673,751],[214,751],[284,751],[502,751]],[[3885],[3886]],[[510]],[[714]],[[3889],[3904],[173],[3882],[3883]],[[3884]],[[3894],[3887],[3888],[3893],[3903]],[[173],[714],[510]],[[19],[29],[46],[47],[58],[61],[677],[75],[77],[90],[123],[147],[159],[196],[197],[219],[222],[234],[245],[267],[373],[415],[417],[455],[468],[480],[489],[512],[514],[543],[552],[597],[606],[607],[651]],[[3891],[3892]],[[3908],[170],[188],[369],[423],[427],[451],[459],[709],[565]],[[3890]],[[3894],[3893],[3903]],[[170],[188],[369],[423],[427],[451],[459],[709],[565]],[[3895],[3897]],[[3],[2],[724],[5],[660],[6],[7],[8],[9],[12],[16],[21],[812],[23],[24],[25],[26],[27],[33],[37],[40],[41],[42],[44],[675],[50],[53],[54],[56],[57],[63],[64],[65],[66],[67],[68],[70],[71],[74],[73],[76],[78],[79],[664],[80],[81],[82],[84],[85],[87],[88],[89],[91],[96],[101],[107],[111],[112],[113],[116],[122],[129],[130],[715],[132],[716],[138],[139],[140],[141],[142],[150],[151],[152],[156],[158],[160],[730],[162],[163],[848],[164],[166],[165],[168],[169],[171],[172],[680],[176],[177],[179],[180],[181],[184],[185],[189],[190],[191],[192],[682],[201],[202],[204],[208],[211],[212],[213],[713],[840],[216],[210],[841],[220],[674],[705],[225],[224],[229],[230],[233],[725],[235],[238],[844],[243],[244],[661],[250],[255],[256],[258],[259],[262],[264],[847],[268],[270],[273],[274],[279],[280],[670],[286],[288],[289],[296],[735],[298],[299],[319],[300],[729],[301],[302],[303],[304],[712],[305],[306],[307],[308],[309],[310],[312],[311],[313],[314],[316],[738],[317],[318],[736],[321],[322],[323],[324],[327],[328],[333],[334],[335],[336],[337],[340],[343],[344],[346],[348],[350],[351],[352],[353],[354],[355],[356],[357],[358],[361],[363],[702],[365],[366],[367],[368],[671],[374],[689],[377],[379],[381],[732],[728],[384],[386],[387],[719],[390],[703],[717],[690],[398],[399],[400],[401],[402],[403],[404],[406],[704],[407],[408],[409],[410],[411],[412],[413],[693],[418],[419],[421],[737],[424],[426],[425],[429],[430],[431],[434],[438],[439],[441],[846],[442],[718],[445],[446],[447],[448],[449],[452],[842],[454],[456],[460],[462],[461],[463],[466],[464],[465],[617],[695],[470],[472],[727],[473],[851],[474],[706],[476],[659],[481],[482],[483],[485],[486],[488],[490],[492],[721],[849],[722],[720],[723],[495],[496],[500],[501],[503],[508],[513],[669],[515],[517],[519],[520],[521],[813],[814],[815],[817],[816],[818],[819],[820],[821],[822],[823],[824],[825],[826],[829],[828],[830],[831],[833],[832],[834],[827],[835],[522],[836],[837],[838],[839],[528],[529],[530],[532],[535],[538],[707],[540],[542],[544],[545],[546],[547],[553],[556],[557],[558],[559],[560],[561],[566],[567],[568],[571],[572],[575],[576],[577],[578],[580],[581],[708],[697],[584],[585],[583],[586],[845],[592],[593],[598],[599],[698],[601],[602],[603],[604],[610],[613],[615],[618],[619],[625],[627],[631],[711],[636],[662],[638],[639],[640],[641],[646],[647],[648],[650],[652],[653],[656],[843]],[[731],[741],[735],[738],[736],[733],[744],[740],[737],[734],[739],[742],[743],[583],[586]],[[3896]],[[3900],[3901]],[[3908],[3904]],[[3899]],[[3894],[3888],[3903]],[[3894],[3887],[3888],[3893]],[[214],[284],[658],[673],[502]],[[3905],[3906],[3907]],[[2],[19],[12],[27],[29],[46],[47],[58],[61],[677],[66],[75],[77],[90],[123],[147],[159],[196],[197],[201],[210],[219],[222],[224],[245],[661],[267],[373],[387],[390],[398],[401],[413],[415],[417],[452],[455],[468],[470],[659],[480],[489],[720],[721],[722],[723],[496],[503],[512],[519],[514],[520],[543],[552],[597],[606],[607],[615],[662],[648],[651]],[[510]],[[234]],[[3909],[3910],[3912],[3914],[3915]],[[3],[724],[5],[6],[7],[8],[9],[13],[16],[21],[22],[24],[23],[25],[26],[33],[37],[40],[42],[41],[44],[675],[50],[53],[54],[56],[57],[63],[65],[64],[67],[68],[70],[73],[74],[71],[76],[78],[79],[664],[80],[81],[82],[84],[85],[87],[89],[88],[91],[93],[96],[101],[107],[112],[111],[113],[116],[122],[129],[130],[132],[136],[716],[138],[139],[140],[141],[142],[150],[151],[152],[158],[160],[164],[163],[162],[165],[166],[168],[169],[171],[680],[176],[179],[180],[181],[185],[184],[682],[202],[156],[204],[189],[190],[191],[192],[208],[212],[211],[213],[216],[214],[220],[674],[705],[225],[229],[230],[233],[250],[235],[238],[244],[725],[255],[256],[258],[259],[243],[262],[850],[264],[268],[270],[273],[274],[279],[280],[284],[670],[286],[288],[289],[323],[316],[319],[300],[304],[301],[302],[318],[303],[712],[306],[298],[305],[299],[314],[308],[307],[317],[309],[310],[311],[312],[313],[296],[321],[322],[325],[324],[327],[328],[333],[334],[335],[336],[337],[340],[343],[344],[348],[346],[350],[351],[352],[353],[354],[355],[357],[356],[358],[361],[363],[702],[365],[367],[366],[374],[368],[689],[671],[377],[379],[381],[728],[382],[384],[719],[703],[717],[690],[399],[400],[402],[403],[404],[406],[704],[407],[409],[410],[408],[411],[412],[693],[418],[419],[708],[421],[424],[425],[426],[429],[430],[431],[434],[438],[439],[441],[440],[442],[445],[446],[447],[448],[449],[676],[454],[456],[460],[461],[462],[463],[464],[465],[466],[617],[695],[472],[727],[473],[474],[706],[476],[481],[482],[483],[485],[486],[488],[490],[492],[495],[501],[500],[502],[508],[513],[669],[515],[517],[521],[522],[528],[529],[530],[533],[532],[535],[538],[707],[540],[542],[544],[545],[546],[547],[553],[556],[557],[558],[559],[561],[560],[565],[566],[567],[568],[576],[571],[575],[572],[577],[578],[580],[581],[697],[592],[593],[583],[584],[585],[586],[598],[599],[600],[698],[601],[602],[604],[603],[610],[613],[618],[619],[631],[711],[636],[627],[639],[638],[640],[647],[641],[650],[652],[653],[656]],[[510]],[[99],[234],[206],[484],[487]],[[3911]],[[172],[177],[386],[565],[625],[646]],[[3913]],[[660]]]]; diff --git a/importer/lib/wp-stubs.php b/importer/lib/wp-stubs.php new file mode 100644 index 00000000..68b07c60 --- /dev/null +++ b/importer/lib/wp-stubs.php @@ -0,0 +1,81 @@ +assertCount(1, $results); + $this->assertEquals($value, $results[0]['value']); + $this->assertFalse($results[0]['is_json']); + } + + public function testScanFindsConvertWrappedValue(): void + { + $value = '{"key": "value"}'; + $encoded = base64_encode($value); + $sql = "INSERT INTO t VALUES(1, CONVERT(FROM_BASE64('{$encoded}') USING utf8mb4));"; + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(1, $results); + $this->assertEquals($value, $results[0]['value']); + $this->assertTrue($results[0]['is_json']); + } + + public function testScanFindsMixedValueTypes(): void + { + $text = 'some text'; + $json = '{"url": "https://example.com"}'; + $text_enc = base64_encode($text); + $json_enc = base64_encode($json); + + $sql = "INSERT INTO t VALUES(1, FROM_BASE64('{$text_enc}'), NULL, 42, CONVERT(FROM_BASE64('{$json_enc}') USING utf8mb4));"; + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(2, $results); + $this->assertEquals($text, $results[0]['value']); + $this->assertFalse($results[0]['is_json']); + $this->assertEquals($json, $results[1]['value']); + $this->assertTrue($results[1]['is_json']); + } + + public function testScanReturnsEmptyForNoBase64(): void + { + $sql = "CREATE TABLE t (id INT, name VARCHAR(255));"; + $results = Base64ValueScanner::scan($sql); + $this->assertCount(0, $results); + } + + public function testScanReturnsEmptyForNullAndNumeric(): void + { + $sql = "INSERT INTO t VALUES(1, NULL, 3.14);"; + $results = Base64ValueScanner::scan($sql); + $this->assertCount(0, $results); + } + + public function testScanOffsetsAreCorrect(): void + { + $value = 'test'; + $encoded = base64_encode($value); + $prefix = "INSERT INTO t VALUES(1, "; + $expr = "FROM_BASE64('{$encoded}')"; + $sql = $prefix . $expr . ", NULL);"; + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(1, $results); + $this->assertEquals(strlen($prefix), $results[0]['offset']); + $this->assertEquals(strlen($expr), $results[0]['length']); + $this->assertEquals($expr, substr($sql, $results[0]['offset'], $results[0]['length'])); + } + + public function testScanConvertOffsetsIncludeWrapper(): void + { + $value = '[]'; + $encoded = base64_encode($value); + $prefix = "INSERT INTO t VALUES("; + $expr = "CONVERT(FROM_BASE64('{$encoded}') USING utf8mb4)"; + $sql = $prefix . $expr . ");"; + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(1, $results); + $this->assertEquals(strlen($prefix), $results[0]['offset']); + $this->assertEquals(strlen($expr), $results[0]['length']); + $this->assertEquals($expr, substr($sql, $results[0]['offset'], $results[0]['length'])); + } + + public function testScanMultipleValuesInOneStatement(): void + { + $v1 = 'alpha'; + $v2 = 'beta'; + $v3 = 'gamma'; + $sql = sprintf( + "INSERT INTO t VALUES(FROM_BASE64('%s'), FROM_BASE64('%s'), FROM_BASE64('%s'));", + base64_encode($v1), + base64_encode($v2), + base64_encode($v3) + ); + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(3, $results); + $this->assertEquals($v1, $results[0]['value']); + $this->assertEquals($v2, $results[1]['value']); + $this->assertEquals($v3, $results[2]['value']); + } + + public function testReplaceSimpleValue(): void + { + $old = 'old value'; + $new = 'new value'; + $encoded_old = base64_encode($old); + $sql = "INSERT INTO t VALUES(FROM_BASE64('{$encoded_old}'));"; + + $results = Base64ValueScanner::scan($sql); + $this->assertCount(1, $results); + + $modified = Base64ValueScanner::replace( + $sql, + $results[0]['offset'], + $results[0]['length'], + $new, + $results[0]['is_json'] + ); + + $expected_encoded = base64_encode($new); + $this->assertStringContainsString("FROM_BASE64('{$expected_encoded}')", $modified); + $this->assertStringNotContainsString($encoded_old, $modified); + } + + public function testReplaceConvertWrappedValue(): void + { + $old = '{"old": true}'; + $new = '{"new": true}'; + $encoded_old = base64_encode($old); + $sql = "INSERT INTO t VALUES(CONVERT(FROM_BASE64('{$encoded_old}') USING utf8mb4));"; + + $results = Base64ValueScanner::scan($sql); + $this->assertCount(1, $results); + + $modified = Base64ValueScanner::replace( + $sql, + $results[0]['offset'], + $results[0]['length'], + $new, + $results[0]['is_json'] + ); + + $expected_encoded = base64_encode($new); + $this->assertStringContainsString("CONVERT(FROM_BASE64('{$expected_encoded}') USING utf8mb4)", $modified); + } + + public function testReplaceInReverseOrderPreservesPositions(): void + { + $v1 = 'first'; + $v2 = 'second'; + $sql = sprintf( + "INSERT INTO t VALUES(FROM_BASE64('%s'), FROM_BASE64('%s'));", + base64_encode($v1), + base64_encode($v2) + ); + + $results = Base64ValueScanner::scan($sql); + $this->assertCount(2, $results); + + // Replace in reverse order to preserve offsets + $modified = $sql; + $modified = Base64ValueScanner::replace( + $modified, + $results[1]['offset'], + $results[1]['length'], + 'SECOND_NEW', + false + ); + $modified = Base64ValueScanner::replace( + $modified, + $results[0]['offset'], + $results[0]['length'], + 'FIRST_NEW', + false + ); + + // Verify both replacements worked + $new_results = Base64ValueScanner::scan($modified); + $this->assertCount(2, $new_results); + $this->assertEquals('FIRST_NEW', $new_results[0]['value']); + $this->assertEquals('SECOND_NEW', $new_results[1]['value']); + } + + public function testScanHandlesEmptyString(): void + { + $value = ''; + $encoded = base64_encode($value); + $sql = "INSERT INTO t VALUES(FROM_BASE64('{$encoded}'));"; + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(1, $results); + $this->assertEquals('', $results[0]['value']); + } + + public function testScanHandlesBase64WithSpecialChars(): void + { + // Value that produces base64 with +, /, and = characters + $value = str_repeat("\xff\xfe\xfd", 10); + $encoded = base64_encode($value); + $sql = "INSERT INTO t VALUES(FROM_BASE64('{$encoded}'));"; + + $results = Base64ValueScanner::scan($sql); + + $this->assertCount(1, $results); + $this->assertEquals($value, $results[0]['value']); + } +} diff --git a/tests/UrlRewriting/ContentClassifierTest.php b/tests/UrlRewriting/ContentClassifierTest.php new file mode 100644 index 00000000..ec1d2afe --- /dev/null +++ b/tests/UrlRewriting/ContentClassifierTest.php @@ -0,0 +1,188 @@ +assertEquals(ContentClassifier::TYPE_SERIALIZED_PHP, ContentClassifier::classify('N;')); + } + + public function testClassifiesSerializedString(): void + { + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize('hello world')) + ); + } + + public function testClassifiesSerializedArray(): void + { + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize(['key' => 'value', 'num' => 42])) + ); + } + + public function testClassifiesSerializedInteger(): void + { + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize(42)) + ); + } + + public function testClassifiesSerializedDouble(): void + { + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize(3.14)) + ); + } + + public function testClassifiesSerializedBoolean(): void + { + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize(true)) + ); + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize(false)) + ); + } + + public function testClassifiesSerializedObject(): void + { + $obj = new stdClass(); + $obj->name = 'test'; + $this->assertEquals( + ContentClassifier::TYPE_SERIALIZED_PHP, + ContentClassifier::classify(serialize($obj)) + ); + } + + public function testClassifiesNestedSerializedArray(): void + { + $data = serialize([ + 'siteurl' => 'https://example.com', + 'nested' => ['a' => 1, 'b' => 2], + ]); + $this->assertEquals(ContentClassifier::TYPE_SERIALIZED_PHP, ContentClassifier::classify($data)); + } + + // --- JSON --- + + public function testClassifiesJsonObject(): void + { + $this->assertEquals( + ContentClassifier::TYPE_JSON, + ContentClassifier::classify('{"key": "value", "num": 42}') + ); + } + + public function testClassifiesJsonArray(): void + { + $this->assertEquals( + ContentClassifier::TYPE_JSON, + ContentClassifier::classify('[1, 2, 3]') + ); + } + + public function testClassifiesNestedJson(): void + { + $json = json_encode([ + 'url' => 'https://example.com', + 'nested' => ['deep' => ['value' => true]], + ]); + $this->assertEquals(ContentClassifier::TYPE_JSON, ContentClassifier::classify($json)); + } + + public function testClassifiesEmptyJsonObject(): void + { + $this->assertEquals(ContentClassifier::TYPE_JSON, ContentClassifier::classify('{}')); + } + + public function testClassifiesEmptyJsonArray(): void + { + $this->assertEquals(ContentClassifier::TYPE_JSON, ContentClassifier::classify('[]')); + } + + // --- Text (HTML, block markup, plain text, etc.) --- + + public function testClassifiesPlainText(): void + { + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('Hello, world!')); + } + + public function testClassifiesHtml(): void + { + $this->assertEquals( + ContentClassifier::TYPE_TEXT, + ContentClassifier::classify('

Visit our site

') + ); + } + + public function testClassifiesBlockMarkup(): void + { + $markup = '
'; + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify($markup)); + } + + public function testClassifiesUrl(): void + { + $this->assertEquals( + ContentClassifier::TYPE_TEXT, + ContentClassifier::classify('https://example.com/page') + ); + } + + public function testClassifiesEmptyString(): void + { + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('')); + } + + // --- Edge cases --- + + public function testStringStartingWithAColonButNotSerialized(): void + { + // "a:not-serialized" starts with 'a:' but is not valid serialized PHP + // It ends with 'd', not ';' or '}' + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('a:not-serialized')); + } + + public function testShortStringsAreText(): void + { + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('hi')); + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('abc')); + } + + public function testInvalidJsonStartingWithBraceIsText(): void + { + // Starts with { but is not valid JSON + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('{not json at all')); + } + + public function testInvalidJsonStartingWithBracketIsText(): void + { + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('[not json')); + } + + public function testJsonStringScalarIsText(): void + { + // A JSON scalar string like "hello" is not classified as JSON + // because it doesn't start with { or [ + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify('"hello"')); + } + + public function testCssIsText(): void + { + $css = 'body { background: url("https://example.com/bg.jpg"); }'; + $this->assertEquals(ContentClassifier::TYPE_TEXT, ContentClassifier::classify($css)); + } +} diff --git a/tests/UrlRewriting/DomainCollectorTest.php b/tests/UrlRewriting/DomainCollectorTest.php new file mode 100644 index 00000000..179c8616 --- /dev/null +++ b/tests/UrlRewriting/DomainCollectorTest.php @@ -0,0 +1,119 @@ +scan('Visit https://example.com/page for details.'); + $domains = $collector->get_domains(); + $this->assertCount(1, $domains); + $this->assertEquals('https://example.com', $domains[0]); + } + + public function testCollectsHttpDomain(): void + { + $collector = new DomainCollector(); + $collector->scan('http://old-site.org/about'); + $domains = $collector->get_domains(); + $this->assertCount(1, $domains); + $this->assertEquals('http://old-site.org', $domains[0]); + } + + public function testCollectsMultipleDomains(): void + { + $collector = new DomainCollector(); + $collector->scan('Links: https://site-a.com/page and https://site-b.com/other'); + $domains = $collector->get_domains(); + $this->assertCount(2, $domains); + $this->assertContains('https://site-a.com', $domains); + $this->assertContains('https://site-b.com', $domains); + } + + public function testDeduplicatesDomains(): void + { + $collector = new DomainCollector(); + $collector->scan('https://example.com/page1 and https://example.com/page2'); + $domains = $collector->get_domains(); + $this->assertCount(1, $domains); + $this->assertEquals('https://example.com', $domains[0]); + } + + public function testCollectsDomainsFromHtml(): void + { + $collector = new DomainCollector(); + $collector->scan('Plugins '); + $domains = $collector->get_domains(); + $this->assertContains('https://wp.org', $domains); + $this->assertContains('https://cdn.wp.org', $domains); + } + + public function testCollectsDomainsAcrossMultipleScans(): void + { + $collector = new DomainCollector(); + $collector->scan('https://first.com/a'); + $collector->scan('https://second.com/b'); + $domains = $collector->get_domains(); + $this->assertCount(2, $domains); + $this->assertContains('https://first.com', $domains); + $this->assertContains('https://second.com', $domains); + } + + public function testMergeWithPreviousList(): void + { + $collector = new DomainCollector(); + $collector->scan('https://new-discovery.com/page'); + $collector->merge(['https://previously-found.com', 'https://another.com']); + $domains = $collector->get_domains(); + $this->assertCount(3, $domains); + $this->assertContains('https://new-discovery.com', $domains); + $this->assertContains('https://previously-found.com', $domains); + $this->assertContains('https://another.com', $domains); + } + + public function testReturnsSortedList(): void + { + $collector = new DomainCollector(); + $collector->scan('https://zebra.com/page https://apple.com/page https://mango.com/page'); + $domains = $collector->get_domains(); + $this->assertEquals(['https://apple.com', 'https://mango.com', 'https://zebra.com'], $domains); + } + + public function testEmptyStringReturnsNoDomains(): void + { + $collector = new DomainCollector(); + $collector->scan(''); + $this->assertCount(0, $collector->get_domains()); + } + + public function testNoUrlsReturnsNoDomains(): void + { + $collector = new DomainCollector(); + $collector->scan('This is just plain text with no URLs.'); + $this->assertCount(0, $collector->get_domains()); + } + + public function testCollectsDomainsWithPorts(): void + { + $collector = new DomainCollector(); + $collector->scan('http://localhost:8080/api and https://dev.example.com:3000/page'); + $domains = $collector->get_domains(); + $this->assertContains('http://localhost:8080', $domains); + $this->assertContains('https://dev.example.com:3000', $domains); + } + + public function testDifferentProtocolsSameDomainAreSeparate(): void + { + $collector = new DomainCollector(); + $collector->scan('http://example.com/a and https://example.com/b'); + $domains = $collector->get_domains(); + $this->assertCount(2, $domains); + $this->assertContains('http://example.com', $domains); + $this->assertContains('https://example.com', $domains); + } +} diff --git a/tests/UrlRewriting/SqlStatementRewriterTest.php b/tests/UrlRewriting/SqlStatementRewriterTest.php new file mode 100644 index 00000000..6040f253 --- /dev/null +++ b/tests/UrlRewriting/SqlStatementRewriterTest.php @@ -0,0 +1,161 @@ + 'https://new-site.com', + ]) + ); + } + + public function testRewritesUrlInInsertStatement(): void + { + $rewriter = $this->createRewriter(); + $html = 'Link'; + $encoded = base64_encode($html); + $sql = "INSERT INTO `wp_posts` VALUES(1, FROM_BASE64('{$encoded}'));"; + + $result = $rewriter->rewrite($sql); + + // Verify the rewritten SQL contains new-site.com + $matches = Base64ValueScanner::scan($result); + $this->assertCount(1, $matches); + $this->assertStringContainsString('new-site.com', $matches[0]['value']); + $this->assertStringNotContainsString('old-site.com', $matches[0]['value']); + } + + public function testPassesThroughDdlStatements(): void + { + $rewriter = $this->createRewriter(); + $sql = "CREATE TABLE `wp_posts` (id INT, content TEXT);"; + $this->assertEquals($sql, $rewriter->rewrite($sql)); + } + + public function testPassesThroughStatementsWithNoBase64(): void + { + $rewriter = $this->createRewriter(); + $sql = "INSERT INTO `wp_posts` VALUES(1, NULL, 42);"; + $this->assertEquals($sql, $rewriter->rewrite($sql)); + } + + public function testSkipsSerializedPhpValues(): void + { + $rewriter = $this->createRewriter(); + $serialized = serialize(['siteurl' => 'https://old-site.com']); + $encoded = base64_encode($serialized); + $sql = "INSERT INTO `wp_options` VALUES(1, FROM_BASE64('{$encoded}'));"; + + $result = $rewriter->rewrite($sql); + + // Should be unchanged — serialized PHP is skipped + $this->assertEquals($sql, $result); + } + + public function testRewritesJsonValues(): void + { + $rewriter = $this->createRewriter(); + $json = json_encode(['url' => 'https://old-site.com/api'], JSON_UNESCAPED_SLASHES); + $encoded = base64_encode($json); + $sql = "INSERT INTO `wp_postmeta` VALUES(1, CONVERT(FROM_BASE64('{$encoded}') USING utf8mb4));"; + + $result = $rewriter->rewrite($sql); + + $matches = Base64ValueScanner::scan($result); + $this->assertCount(1, $matches); + $this->assertTrue($matches[0]['is_json']); + $decoded = json_decode($matches[0]['value'], true); + $this->assertStringContainsString('new-site.com', $decoded['url']); + } + + public function testHandlesMixedValueTypes(): void + { + $rewriter = $this->createRewriter(); + + $html = '

Visit us

'; + $serialized = serialize(['url' => 'https://old-site.com']); + $plain = 'https://old-site.com/about'; + + $sql = sprintf( + "INSERT INTO `t` VALUES(1, FROM_BASE64('%s'), FROM_BASE64('%s'), NULL, FROM_BASE64('%s'));", + base64_encode($html), + base64_encode($serialized), + base64_encode($plain) + ); + + $result = $rewriter->rewrite($sql); + + $matches = Base64ValueScanner::scan($result); + $this->assertCount(3, $matches); + + // HTML should be rewritten + $this->assertStringContainsString('new-site.com', $matches[0]['value']); + + // Serialized PHP should be unchanged + $this->assertEquals($serialized, $matches[1]['value']); + + // Plain text should be rewritten + $this->assertStringContainsString('new-site.com', $matches[2]['value']); + } + + public function testValuesWithNoMatchingUrlsAreUnchanged(): void + { + $rewriter = $this->createRewriter(); + $text = 'No URLs here, just plain text.'; + $encoded = base64_encode($text); + $sql = "INSERT INTO `t` VALUES(FROM_BASE64('{$encoded}'));"; + + $result = $rewriter->rewrite($sql); + + $matches = Base64ValueScanner::scan($result); + $this->assertCount(1, $matches); + $this->assertEquals($text, $matches[0]['value']); + } + + public function testRewritesMultipleRowInsert(): void + { + $rewriter = $this->createRewriter(); + $url1 = 'https://old-site.com/page1'; + $url2 = 'https://old-site.com/page2'; + + $sql = sprintf( + "INSERT INTO `t` VALUES(1, FROM_BASE64('%s')), (2, FROM_BASE64('%s'));", + base64_encode($url1), + base64_encode($url2) + ); + + $result = $rewriter->rewrite($sql); + + $matches = Base64ValueScanner::scan($result); + $this->assertCount(2, $matches); + $this->assertStringContainsString('new-site.com/page1', $matches[0]['value']); + $this->assertStringContainsString('new-site.com/page2', $matches[1]['value']); + } + + public function testResultIsValidSql(): void + { + $rewriter = $this->createRewriter(); + $html = ''; + $encoded = base64_encode($html); + $sql = "INSERT INTO `wp_posts` (`id`, `content`) VALUES(1, FROM_BASE64('{$encoded}'));"; + + $result = $rewriter->rewrite($sql); + + // Verify the result still has proper SQL structure + $this->assertStringStartsWith('INSERT INTO', $result); + $this->assertStringEndsWith(');', $result); + $this->assertStringContainsString("FROM_BASE64('", $result); + $this->assertStringContainsString("')", $result); + } +} diff --git a/tests/UrlRewriting/SqlValueUrlRewriterTest.php b/tests/UrlRewriting/SqlValueUrlRewriterTest.php new file mode 100644 index 00000000..5889e4f8 --- /dev/null +++ b/tests/UrlRewriting/SqlValueUrlRewriterTest.php @@ -0,0 +1,177 @@ + 'https://new-site.com', + ]); + } + + // --- HTML content --- + + public function testRewritesUrlInHrefAttribute(): void + { + $rewriter = $this->createRewriter(); + $input = 'Link'; + $result = $rewriter->rewrite($input); + $this->assertStringContainsString('https://new-site.com/page', $result); + $this->assertStringNotContainsString('old-site.com', $result); + } + + public function testRewritesUrlInImgSrc(): void + { + $rewriter = $this->createRewriter(); + $input = ''; + $result = $rewriter->rewrite($input); + $this->assertStringContainsString('https://new-site.com/wp-content/uploads/photo.jpg', $result); + } + + public function testRewritesMultipleHtmlAttributes(): void + { + $rewriter = $this->createRewriter(); + $input = 'Link 1Link 2'; + $result = $rewriter->rewrite($input); + $this->assertStringContainsString('https://new-site.com/page1', $result); + $this->assertStringContainsString('https://new-site.com/page2', $result); + } + + // --- Block markup --- + + public function testRewritesBlockMarkupJsonAttributes(): void + { + $rewriter = $this->createRewriter(); + $input = '
'; + $result = $rewriter->rewrite($input); + $this->assertStringNotContainsString('old-site.com', $result); + $this->assertStringContainsString('new-site.com', $result); + } + + // --- Plain text with URLs --- + + public function testRewritesBareUrlInText(): void + { + $rewriter = $this->createRewriter(); + $input = 'Visit us at https://old-site.com/about for more info.'; + $result = $rewriter->rewrite($input); + $this->assertStringContainsString('https://new-site.com/about', $result); + } + + // --- JSON content --- + + public function testRewritesUrlsInJsonStringValues(): void + { + $rewriter = $this->createRewriter(); + $input = json_encode([ + 'home' => 'https://old-site.com', + 'logo' => 'https://old-site.com/wp-content/uploads/logo.png', + ], JSON_UNESCAPED_SLASHES); + + $result = $rewriter->rewrite($input); + $decoded = json_decode($result, true); + + $this->assertNotNull($decoded); + $this->assertStringContainsString('new-site.com', $decoded['home']); + $this->assertStringContainsString('new-site.com/wp-content/uploads/logo.png', $decoded['logo']); + } + + public function testRewritesUrlsInNestedJson(): void + { + $rewriter = $this->createRewriter(); + $input = json_encode([ + 'settings' => [ + 'url' => 'https://old-site.com/api', + 'nested' => [ + 'image' => 'https://old-site.com/img.jpg', + ], + ], + 'count' => 42, + 'active' => true, + ], JSON_UNESCAPED_SLASHES); + + $result = $rewriter->rewrite($input); + $decoded = json_decode($result, true); + + $this->assertNotNull($decoded); + $this->assertStringContainsString('new-site.com', $decoded['settings']['url']); + $this->assertStringContainsString('new-site.com', $decoded['settings']['nested']['image']); + $this->assertEquals(42, $decoded['count']); + $this->assertTrue($decoded['active']); + } + + public function testJsonOutputUsesUnescapedSlashes(): void + { + $rewriter = $this->createRewriter(); + $input = '{"url":"https://old-site.com/path"}'; + $result = $rewriter->rewrite($input); + // Should not contain escaped slashes like \/ + $this->assertStringNotContainsString('\\/', $result); + } + + // --- Serialized PHP --- + + public function testSerializedPhpIsReturnedUnchanged(): void + { + $rewriter = $this->createRewriter(); + $input = serialize([ + 'siteurl' => 'https://old-site.com', + 'blogname' => 'My Old Site', + ]); + $result = $rewriter->rewrite($input); + $this->assertEquals($input, $result, 'Serialized PHP should be returned unchanged'); + } + + public function testSerializedStringIsReturnedUnchanged(): void + { + $rewriter = $this->createRewriter(); + $input = serialize('https://old-site.com'); + $result = $rewriter->rewrite($input); + $this->assertEquals($input, $result); + } + + // --- No-change cases --- + + public function testValueWithNoUrlsReturnsUnchanged(): void + { + $rewriter = $this->createRewriter(); + $input = 'Just a regular string with no URLs.'; + $result = $rewriter->rewrite($input); + $this->assertEquals($input, $result); + } + + public function testEmptyStringReturnsUnchanged(): void + { + $rewriter = $this->createRewriter(); + $this->assertEquals('', $rewriter->rewrite('')); + } + + public function testUrlFromDifferentDomainIsNotRewritten(): void + { + $rewriter = $this->createRewriter(); + $input = 'Link'; + $result = $rewriter->rewrite($input); + $this->assertStringContainsString('other-site.com', $result); + $this->assertStringNotContainsString('new-site.com', $result); + } + + // --- Multiple URL mappings --- + + public function testMultipleUrlMappings(): void + { + $rewriter = $this->createRewriter([ + 'https://old-site.com' => 'https://new-site.com', + 'https://cdn.old-site.com' => 'https://cdn.new-site.com', + ]); + $input = 'Link'; + $result = $rewriter->rewrite($input); + $this->assertStringContainsString('cdn.new-site.com', $result); + $this->assertStringContainsString('new-site.com/page', $result); + } +} diff --git a/tests/e2e/Dockerfile b/tests/e2e/Dockerfile index 077a05a6..e4b215fc 100644 --- a/tests/e2e/Dockerfile +++ b/tests/e2e/Dockerfile @@ -34,10 +34,16 @@ RUN groupadd -f nginx && useradd -r -g nginx -s /usr/sbin/nologin nginx 2>/dev/n RUN curl -sfL "https://raw.githubusercontent.com/wp-cli/builds/gh-pages/phar/wp-cli.phar" -o /tmp/wp-cli.phar \ && chmod +x /tmp/wp-cli.phar +# Composer +RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer + # Copy project COPY . /app WORKDIR /app +# Install PHP dependencies (vendor/ is gitignored) +RUN composer install --no-dev --no-interaction --prefer-dist + # Install npm deps for E2E RUN cd tests/e2e && npm install --silent diff --git a/tests/e2e/setup.sh b/tests/e2e/setup.sh index b8a72adc..12b74375 100755 --- a/tests/e2e/setup.sh +++ b/tests/e2e/setup.sh @@ -1,4 +1,12 @@ #!/usr/bin/env bash set -euo pipefail + +# Install Composer dependencies (vendor/ is gitignored). +# Needed for url-rewriting classes from wp-php-toolkit/data-liberation. +PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +if command -v composer &>/dev/null && [ -f "$PROJECT_ROOT/composer.json" ]; then + composer install --no-dev --no-interaction --prefer-dist --working-dir="$PROJECT_ROOT" +fi + cd "$(dirname "$0")" npm install --silent diff --git a/tests/e2e/site-registry.json b/tests/e2e/site-registry.json index af95bc90..dd335209 100644 --- a/tests/e2e/site-registry.json +++ b/tests/e2e/site-registry.json @@ -30,6 +30,7 @@ "large-single-file": { "port": 8103 }, "file-deletions": { "port": 8104 }, "file-type-swaps": { "port": 8105 }, - "file-type-cache": { "port": 8106 } + "file-type-cache": { "port": 8106 }, + "url-rewriting": { "port": 8107 } } } diff --git a/tests/e2e/tests/import-36-url-rewriting.test.js b/tests/e2e/tests/import-36-url-rewriting.test.js new file mode 100644 index 00000000..3dc9c0e1 --- /dev/null +++ b/tests/e2e/tests/import-36-url-rewriting.test.js @@ -0,0 +1,263 @@ +/** + * Test 36: URL Rewriting via db-apply + * + * Tests the full round-trip: + * 1. Create site with known content containing source URLs in various formats + * 2. Run db-sync → verify .import-domains.json contains the source domain + * 3. Run db-apply with --url-mapping to apply SQL to target database + * 4. Verify URLs are rewritten in non-serialized values, serialized PHP is unchanged + */ +import { describe, it, beforeAll, afterAll } from 'vitest'; +import assert from 'node:assert/strict'; +import { readFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { + runImporter, createTempDir, cleanupTempDir, + getSiteUrl, getSiteSecret, getSiteDir, + getDbName, createMysqlConnection, +} from '../lib/test-helpers.js'; +import { ensureSite } from '../lib/site-setup.js'; + +describe('Import: URL Rewriting', () => { + const site = 'url-rewriting'; + const importDb = 'e2e_url_rewriting_import_36'; + let tempDir; + // Must match what wp core install sets (http://127.0.0.1:PORT) + const SOURCE_DOMAIN = 'http://127.0.0.1:8107'; + const TARGET_DOMAIN = 'https://target.example.com'; + + beforeAll(async () => { + await ensureSite(site, { + customDb: async (dbName, conn) => { + // WordPress tables already exist from wp core install. + // Just INSERT additional test data into existing tables. + + // HTML content with URLs (new option_name, no conflict) + await conn.query( + `INSERT INTO wp_options (option_name, option_value) VALUES (?, ?)`, + ['html_option', `About `] + ); + + // Serialized PHP with URLs (should NOT be rewritten) + const serialized = `a:2:{s:7:"siteurl";s:${SOURCE_DOMAIN.length}:"${SOURCE_DOMAIN}";s:4:"home";s:${SOURCE_DOMAIN.length}:"${SOURCE_DOMAIN}";}`; + await conn.query( + `INSERT INTO wp_options (option_name, option_value) VALUES (?, ?)`, + ['serialized_option', serialized] + ); + + // Insert a post with block markup and plain text URLs + const blockMarkup = `
`; + await conn.query( + `INSERT INTO wp_posts (post_author, post_date, post_date_gmt, post_content, post_title, post_excerpt, post_status, comment_status, ping_status, post_password, post_name, to_ping, pinged, post_modified, post_modified_gmt, post_content_filtered, post_parent, guid, menu_order, post_type, post_mime_type, comment_count) VALUES (1, NOW(), NOW(), ?, 'URL Rewrite Test Post', ?, 'publish', 'open', 'open', '', 'url-rewrite-test', '', '', NOW(), NOW(), '', 0, ?, 0, 'post', '', 0)`, + [blockMarkup, `Visit ${SOURCE_DOMAIN}/blog for more`, `${SOURCE_DOMAIN}/?p=999`] + ); + + // Get the ID of the post we just inserted + const [[{ id: postId }]] = await conn.query( + `SELECT LAST_INSERT_ID() as id` + ); + + // Plain URL in meta_value + await conn.query( + `INSERT INTO wp_postmeta (post_id, meta_key, meta_value) VALUES (?, ?, ?)`, + [postId, '_plain_url', `${SOURCE_DOMAIN}/some-page`] + ); + + // Value with no URLs (should be unchanged) + await conn.query( + `INSERT INTO wp_postmeta (post_id, meta_key, meta_value) VALUES (?, ?, ?)`, + [postId, '_no_urls', 'Just a regular string with no URLs'] + ); + }, + }); + tempDir = createTempDir('e2e-url-rewriting'); + const conn = await createMysqlConnection(); + await conn.query(`DROP DATABASE IF EXISTS \`${importDb}\``); + await conn.end(); + }); + + afterAll(async () => { + cleanupTempDir(tempDir); + const conn = await createMysqlConnection(); + await conn.query(`DROP DATABASE IF EXISTS \`${importDb}\``); + await conn.end(); + }); + + function importUrl() { + return `${getSiteUrl(site)}&directory=${getSiteDir(site)}`; + } + + it('db-sync completes and produces db.sql', () => { + const result = runImporter(importUrl(), tempDir, 'db-sync', { + secret: getSiteSecret(site), + }); + assert.equal(result.exitCode, 0, + `Expected exit 0, got ${result.exitCode}\nstderr: ${result.stderr}\nstdout: ${result.stdout}`); + + const sqlFile = join(tempDir, 'db.sql'); + assert.ok(existsSync(sqlFile), 'Expected db.sql to exist'); + }); + + it('domain discovery produces .import-domains.json', () => { + const domainsFile = join(tempDir, '.import-domains.json'); + assert.ok(existsSync(domainsFile), 'Expected .import-domains.json to exist'); + + const domains = JSON.parse(readFileSync(domainsFile, 'utf-8')); + assert.ok(Array.isArray(domains), 'Expected domains to be an array'); + assert.ok(domains.length > 0, 'Expected at least one domain'); + assert.ok( + domains.some(d => d.includes('127.0.0.1:8107')), + `Expected to find 127.0.0.1:8107 in domains, got: ${JSON.stringify(domains)}` + ); + }); + + it('db-apply with URL mapping rewrites URLs in target database', async () => { + // Create target database + const conn = await createMysqlConnection(); + await conn.query(`CREATE DATABASE \`${importDb}\``); + await conn.end(); + + // Run db-apply with URL mapping + const result = runImporter(importUrl(), tempDir, 'db-apply', { + secret: getSiteSecret(site), + extraArgs: [ + `--target-user=e2e_admin`, + `--target-pass=e2e_password`, + `--target-db=${importDb}`, + `--url-mapping=${SOURCE_DOMAIN}::${TARGET_DOMAIN}`, + ], + }); + + assert.equal(result.exitCode, 0, + `Expected exit 0, got ${result.exitCode}\nstderr: ${result.stderr}\nstdout: ${result.stdout}`); + }); + + it('siteurl and home options are rewritten', async () => { + const conn = await createMysqlConnection(importDb); + const [[siteurl]] = await conn.query( + "SELECT option_value FROM wp_options WHERE option_name = 'siteurl'" + ); + const [[home]] = await conn.query( + "SELECT option_value FROM wp_options WHERE option_name = 'home'" + ); + await conn.end(); + + assert.ok(siteurl, 'Expected siteurl row'); + assert.ok(home, 'Expected home row'); + assert.ok( + siteurl.option_value.includes('target.example.com'), + `Expected siteurl to contain target domain, got: ${siteurl.option_value}` + ); + assert.ok( + home.option_value.includes('target.example.com'), + `Expected home to contain target domain, got: ${home.option_value}` + ); + assert.ok( + !siteurl.option_value.includes('127.0.0.1:8107'), + `Expected siteurl to NOT contain source domain, got: ${siteurl.option_value}` + ); + }); + + it('HTML option URLs are rewritten', async () => { + const conn = await createMysqlConnection(importDb); + const [[row]] = await conn.query( + "SELECT option_value FROM wp_options WHERE option_name = 'html_option'" + ); + await conn.end(); + + assert.ok(row, 'Expected html_option row'); + assert.ok( + row.option_value.includes('target.example.com/about'), + `Expected rewritten href, got: ${row.option_value}` + ); + assert.ok( + row.option_value.includes('target.example.com/logo.png'), + `Expected rewritten img src, got: ${row.option_value}` + ); + assert.ok( + !row.option_value.includes('127.0.0.1:8107'), + `Expected no source domain in HTML, got: ${row.option_value}` + ); + }); + + it('serialized PHP values are NOT rewritten', async () => { + const conn = await createMysqlConnection(importDb); + const [[row]] = await conn.query( + "SELECT option_value FROM wp_options WHERE option_name = 'serialized_option'" + ); + await conn.end(); + + assert.ok(row, 'Expected serialized_option row'); + // Serialized PHP should still contain the source domain + assert.ok( + row.option_value.includes('127.0.0.1:8107'), + `Expected serialized PHP to still contain source domain (unchanged), got: ${row.option_value}` + ); + // Verify it still starts with serialized PHP format + assert.ok( + row.option_value.startsWith('a:'), + `Expected serialized PHP format, got: ${row.option_value.substring(0, 10)}` + ); + }); + + it('block markup URLs are rewritten', async () => { + const conn = await createMysqlConnection(importDb); + const [[row]] = await conn.query( + "SELECT post_content FROM wp_posts WHERE post_name = 'url-rewrite-test'" + ); + await conn.end(); + + assert.ok(row, 'Expected url-rewrite-test post'); + assert.ok( + row.post_content.includes('target.example.com'), + `Expected block markup to contain target domain, got: ${row.post_content}` + ); + assert.ok( + !row.post_content.includes('127.0.0.1:8107'), + `Expected block markup to NOT contain source domain, got: ${row.post_content}` + ); + }); + + it('plain text URLs in post_excerpt are rewritten', async () => { + const conn = await createMysqlConnection(importDb); + const [[row]] = await conn.query( + "SELECT post_excerpt FROM wp_posts WHERE post_name = 'url-rewrite-test'" + ); + await conn.end(); + + assert.ok(row, 'Expected url-rewrite-test post'); + assert.ok( + row.post_excerpt.includes('target.example.com/blog'), + `Expected excerpt to contain rewritten URL, got: ${row.post_excerpt}` + ); + }); + + it('plain URL meta values are rewritten', async () => { + const conn = await createMysqlConnection(importDb); + const [[row]] = await conn.query( + "SELECT meta_value FROM wp_postmeta WHERE meta_key = '_plain_url'" + ); + await conn.end(); + + assert.ok(row, 'Expected _plain_url meta row'); + assert.ok( + row.meta_value.includes('target.example.com/some-page'), + `Expected meta URL to be rewritten, got: ${row.meta_value}` + ); + }); + + it('values with no URLs are unchanged', async () => { + const conn = await createMysqlConnection(importDb); + const [[row]] = await conn.query( + "SELECT meta_value FROM wp_postmeta WHERE meta_key = '_no_urls'" + ); + await conn.end(); + + assert.ok(row, 'Expected _no_urls meta row'); + assert.equal( + row.meta_value, + 'Just a regular string with no URLs', + `Expected unchanged value, got: ${row.meta_value}` + ); + }); +}); diff --git a/tests/phpunit.xml b/tests/phpunit.xml index 677577d5..6dc954b8 100644 --- a/tests/phpunit.xml +++ b/tests/phpunit.xml @@ -16,6 +16,9 @@ Import + + UrlRewriting + BuildPdoDsnTest.php