diff --git a/.github/workflows/pr-benchmark-comment.yml b/.github/workflows/pr-benchmark-comment.yml new file mode 100644 index 000000000..4c960a05b --- /dev/null +++ b/.github/workflows/pr-benchmark-comment.yml @@ -0,0 +1,38 @@ +name: PR Benchmark Comment + +on: + workflow_run: + workflows: ["PR Benchmark"] + types: [completed] + +permissions: + pull-requests: write + +jobs: + comment: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Download benchmark result + uses: actions/download-artifact@v4 + with: + name: benchmark-result + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Read PR number and body + id: data + run: | + echo "pr=$(cat pr-number.txt)" >> "$GITHUB_OUTPUT" + { + echo "body<> "$GITHUB_OUTPUT" + + - name: Comment on PR + uses: thollander/actions-comment-pull-request@v3 + with: + pr-number: ${{ steps.data.outputs.pr }} + message: ${{ steps.data.outputs.body }} + comment-tag: benchmark-report diff --git a/.github/workflows/pr-benchmark.yml b/.github/workflows/pr-benchmark.yml index 7a417387c..c0707d616 100644 --- a/.github/workflows/pr-benchmark.yml +++ b/.github/workflows/pr-benchmark.yml @@ -2,7 +2,6 @@ name: PR Benchmark permissions: contents: read - pull-requests: write on: pull_request: @@ -19,7 +18,6 @@ jobs: contains(github.event.comment.body, 'retrigger-benchmark')) runs-on: ubuntu-latest steps: - # Check out the PR head so comment-triggered runs benchmark the PR, not main. - uses: actions/checkout@v6 with: ref: refs/pull/${{ github.event.pull_request.number || github.event.issue.number }}/head @@ -31,8 +29,7 @@ jobs: gui-deps: "false" - name: Create dummy Minecraft world directory - run: | - mkdir -p "./world/region" + run: mkdir -p "./world/region" - name: Build for release run: cargo build --release --no-default-features @@ -79,7 +76,6 @@ jobs: diff=$((duration - baseline_time)) abs_diff=${diff#-} - # Use generation-only time for verdict when available if [ -n "$gen_time" ]; then eval_diff=$((gen_time - baseline_time)) else @@ -146,10 +142,17 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" - - name: Comment build time on PR - uses: thollander/actions-comment-pull-request@v3 + - name: Save benchmark result to files + run: | + cat > benchmark-comment.md <<'COMMENT_EOF' + ${{ steps.comment_body.outputs.summary }} + COMMENT_EOF + echo "${{ github.event.pull_request.number || github.event.issue.number }}" > pr-number.txt + + - name: Upload benchmark result + uses: actions/upload-artifact@v4 with: - message: ${{ steps.comment_body.outputs.summary }} - comment-tag: benchmark-report - env: - GITHUB_TOKEN: ${{ secrets.BENCHMARK_TOKEN }} + name: benchmark-result + path: | + benchmark-comment.md + pr-number.txt diff --git a/src/args.rs b/src/args.rs index 2f8f8db3d..ec4a57008 100644 --- a/src/args.rs +++ b/src/args.rs @@ -40,6 +40,18 @@ pub struct Args { #[arg(long, default_value_t = 1.0)] pub scale: f64, + /// Percentage of CPU cores to use for world generation (10-100). + /// Higher values spawn more worker threads: faster generation, but the + /// system stays less responsive. Overridden by the RAYON_NUM_THREADS env var. + #[arg(long, default_value_t = 90)] + pub cpu_percent: u8, + + /// Overlap each tile batch's merge with the next batch's placement so worker + /// cores aren't idle during merges (faster on large areas; output unchanged). + /// Also enabled by the ARNIS_PIPELINE_MERGE env var. + #[arg(long, default_value_t = false)] + pub pipeline_merge: bool, + /// Projection mode for coordinate mapping /// local: each generation starts at Minecraft (0,0) (default) /// web_mercator: global projection for multi-generation worlds @@ -194,6 +206,11 @@ pub fn validate_args(args: &Args) -> Result<(), String> { return Err("Rotation angle must be between -90 and 90 degrees.".to_string()); } + // Validate CPU usage percentage + if args.cpu_percent < 10 || args.cpu_percent > 100 { + return Err("CPU usage percent (--cpu-percent) must be between 10 and 100.".to_string()); + } + Ok(()) } diff --git a/src/data_processing.rs b/src/data_processing.rs index b6e209a11..3df7de35d 100644 --- a/src/data_processing.rs +++ b/src/data_processing.rs @@ -10,7 +10,7 @@ use crate::progress::{emit_gui_progress_update, emit_gui_progress_update_ex, emi #[cfg(feature = "gui")] use crate::telemetry::{send_log, LogLevel}; use crate::tile; -use crate::world_editor::{FlushWorker, WorldEditor, WorldFormat}; +use crate::world_editor::{FlushWorker, WorldEditor, WorldFormat, WorldToModify}; use colored::Colorize; use indicatif::{ProgressBar, ProgressStyle}; use rayon::prelude::*; @@ -253,6 +253,18 @@ fn should_stream_to_disk(num_regions: usize) -> bool { } /// Generate world with explicit format options (used by GUI for Bedrock support) +/// Public entry point. Runs the whole generation inside a dedicated rayon +/// thread pool sized to the user's CPU budget (`args.cpu_percent`). +/// +/// A *scoped* pool is used instead of reconfiguring the global one because +/// rayon's global pool can only be built once per process (it's already built +/// at startup), so a per-run scoped pool is the only way to let the setting +/// take effect on every generation without restarting the app. Every nested +/// `par_iter` inside `generate_world_inner` (tile placement, region saving, +/// flood-fill precompute, …) automatically runs on this pool via `install`. +/// +/// `RAYON_NUM_THREADS`, if set, wins (power-user / benchmark override). If the +/// pool can't be built we fall back to running on the global pool. pub fn generate_world_with_options( elements: Vec, xzbbox: XZBBox, @@ -261,6 +273,64 @@ pub fn generate_world_with_options( args: &Args, options: GenerationOptions, outline_suppression: OutlineSuppression, +) -> Result { + let target_threads = generation_thread_count(args.cpu_percent); + + match rayon::ThreadPoolBuilder::new() + .num_threads(target_threads) + .thread_name(|i| format!("arnis-gen-{i}")) + .build() + { + Ok(pool) => pool.install(move || { + generate_world_inner( + elements, + xzbbox, + llbbox, + ground, + args, + options, + outline_suppression, + ) + }), + Err(_) => generate_world_inner( + elements, + xzbbox, + llbbox, + ground, + args, + options, + outline_suppression, + ), + } +} + +/// Resolve the worker-thread count for generation from a CPU percentage. +/// `RAYON_NUM_THREADS` overrides the percentage when set; otherwise the count +/// is `round(cores * percent/100)`, clamped to at least 1. +fn generation_thread_count(cpu_percent: u8) -> usize { + if let Some(n) = std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&n| n > 0) + { + return n; + } + + let cores = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + let fraction = f64::from(cpu_percent.clamp(10, 100)) / 100.0; + ((cores as f64 * fraction).round() as usize).max(1) +} + +fn generate_world_inner( + elements: Vec, + xzbbox: XZBBox, + llbbox: LLBBox, + ground: Ground, + args: &Args, + options: GenerationOptions, + outline_suppression: OutlineSuppression, ) -> Result { let output_path = options.path.clone(); let world_format = options.format; @@ -461,10 +531,12 @@ pub fn generate_world_with_options( let total_tiles = indexed_tiles.len().max(1); let mut tiles_merged = 0usize; let mut last_emitted_pct = 20.0_f64; - for batch in indexed_tiles.chunks(tile_batch_size) { - // Phase 1: process this batch of tiles in parallel - let place_start = std::time::Instant::now(); - let batch_results: Vec<_> = batch + + // Phase 1 (parallel): place one batch of tiles into independent tile editors. + // Pure function of immutable inputs — never touches the shared `editor`, so it + // can run concurrently with a Phase-2 merge of an earlier batch. + let place_batch = |batch: &[(usize, &tile::TileBounds)]| { + batch .par_iter() .map(|&(tile_idx, tile_bounds)| { // max_* are exclusive; rect_from_min_max treats max as inclusive, @@ -568,12 +640,21 @@ pub fn generate_world_with_options( tile_road_overrides, ) }) - .collect(); - place_dur += place_start.elapsed(); + .collect::>() + }; - let merge_start = std::time::Instant::now(); - // Phase 2: merge this batch's results into the main editor (sequential). - // batch_results is dropped after this loop, freeing memory before next batch. + // Phase 2 (sequential): merge a batch's tile results into the shared `editor`, + // in the batch's authoritative order. This is the only writer of `editor` and + // the eviction bookkeeping, so the merge order is fixed regardless of whether a + // placement is overlapping — output stays bit-exact. + #[allow(clippy::type_complexity)] + let mut merge_batch = |batch_results: Vec<( + usize, + WorldToModify, + Vec<(i32, i32)>, + fnv::FnvHashMap<(i32, i32), i32>, + )>| + -> Result<(), String> { for (tile_idx, tile_world, tile_subway_pts, tile_road_overrides) in batch_results { editor.merge_world( tile_world, @@ -632,7 +713,46 @@ pub fn generate_world_with_options( last_emitted_pct = pct; } } - merge_dur += merge_start.elapsed(); + Ok(()) + }; + + // Opt-in (GUI "Faster merge" toggle / --pipeline-merge / ARNIS_PIPELINE_MERGE env): + // overlap the serial merge of batch N with the parallel placement of batch N+1 via + // rayon::join, so worker cores aren't idle during merges. Merge order is unchanged + // (in-order, batch by batch), so output is bit-exact vs. the default path — verify + // with --benchmark + ARNIS_BLOCK_HASH. + let pipeline_merge = + args.pipeline_merge || std::env::var_os("ARNIS_PIPELINE_MERGE").is_some(); + if pipeline_merge { + let loop_start = std::time::Instant::now(); + let mut pending: Option> = None; + for batch in indexed_tiles.chunks(tile_batch_size) { + if let Some(prev) = pending.take() { + // Place this batch while the previous batch merges. + let (this_results, merge_res) = + rayon::join(|| place_batch(batch), || merge_batch(prev)); + merge_res?; + pending = Some(this_results); + } else { + pending = Some(place_batch(batch)); + } + } + if let Some(prev) = pending.take() { + merge_batch(prev)?; + } + // Overlapped: report wall-clock as placement, merge folded in. + place_dur = loop_start.elapsed(); + } else { + // Default: place a batch, then merge it, repeat (cores idle during merge). + for batch in indexed_tiles.chunks(tile_batch_size) { + let place_start = std::time::Instant::now(); + let batch_results = place_batch(batch); + place_dur += place_start.elapsed(); + + let merge_start = std::time::Instant::now(); + merge_batch(batch_results)?; + merge_dur += merge_start.elapsed(); + } } bench.report("element_placement", place_dur); bench.report("tile_merge", merge_dur); diff --git a/src/gui.rs b/src/gui.rs index b21838057..68a25005b 100644 --- a/src/gui.rs +++ b/src/gui.rs @@ -853,6 +853,8 @@ fn gui_start_generation( telemetry_consent: bool, world_format: String, rotation_angle: f64, + cpu_percent: u8, + pipeline_merge: bool, ) -> Result<(), String> { use progress::emit_gui_error; use LLBBox; @@ -1103,6 +1105,8 @@ fn gui_start_generation( luanti: world_format == WorldFormat::LuantiWorld, downloader: "requests".to_string(), scale: world_scale, + cpu_percent: cpu_percent.clamp(10, 100), + pipeline_merge, projection: crate::projection::ProjectionKind::Local, ground_level, terrain: terrain_enabled, diff --git a/src/gui/index.html b/src/gui/index.html index 6635e93ba..33052732b 100644 --- a/src/gui/index.html +++ b/src/gui/index.html @@ -79,6 +79,30 @@

Settings

+ + +
+ +
+ + 90% +
+
+ + +
+ +
+ +
+
+
Generation
diff --git a/src/gui/js/main.js b/src/gui/js/main.js index 1cb3b85ac..49e7150c5 100644 --- a/src/gui/js/main.js +++ b/src/gui/js/main.js @@ -104,6 +104,7 @@ async function applyLocalization(localization) { "#world-name-label[data-placeholder]": "no_world_generated_yet", "h2[data-localize='customization_settings']": "customization_settings", "span[data-localize='world_scale']": "world_scale", + "span[data-localize='cpu_usage']": "cpu_usage", "span[data-localize='custom_bounding_box']": "custom_bounding_box", // DEPRECATED: Ground level localization removed // "label[data-localize='ground_level']": "ground_level", @@ -121,6 +122,7 @@ async function applyLocalization(localization) { "span[data-localize='disable_height_limit']": "disable_height_limit", "span[data-localize='aws_only_elevation']": "aws_only_elevation", "span[data-localize='bake_lighting']": "bake_lighting", + "span[data-localize='pipeline_merge']": "pipeline_merge", "span[data-localize='anonymous_crash_reports']": "anonymous_crash_reports", "span[data-localize='map_theme']": "map_theme", "span[data-localize='save_path']": "save_path", @@ -716,6 +718,37 @@ function initSettings() { sliderValue.textContent = "1.00"; }); + // CPU usage slider: percentage of CPU cores used for world generation. + // Persisted to localStorage so the choice survives restarts. + const cpuSlider = document.getElementById("cpu-usage-slider"); + const cpuValue = document.getElementById("cpu-usage-value"); + if (cpuSlider && cpuValue) { + const savedCpu = localStorage.getItem("arnis-cpu-percent"); + if (savedCpu !== null && !isNaN(parseInt(savedCpu, 10))) { + cpuSlider.value = Math.min(Math.max(parseInt(savedCpu, 10), 10), 100); + } + cpuValue.textContent = `${parseInt(cpuSlider.value, 10)}%`; + cpuSlider.addEventListener("input", () => { + cpuValue.textContent = `${parseInt(cpuSlider.value, 10)}%`; + localStorage.setItem("arnis-cpu-percent", cpuSlider.value); + }); + // Double-click to reset CPU usage to default (90%) + cpuSlider.addEventListener("dblclick", () => { + cpuSlider.value = 90; + cpuValue.textContent = "90%"; + localStorage.setItem("arnis-cpu-percent", "90"); + }); + } + + // "Faster merge" toggle: persisted so the choice sticks across launches. + const pipelineToggle = document.getElementById("pipeline-merge-toggle"); + if (pipelineToggle) { + pipelineToggle.checked = localStorage.getItem("arnis-pipeline-merge") === "1"; + pipelineToggle.addEventListener("change", () => { + localStorage.setItem("arnis-pipeline-merge", pipelineToggle.checked ? "1" : "0"); + }); + } + // Rotation angle input const rotationInput = document.getElementById("rotation-angle-input"); @@ -1576,7 +1609,9 @@ async function startGeneration() { var disable_height_limit = document.getElementById("disable-height-limit-toggle").checked; var aws_only_elevation = document.getElementById("aws-only-elevation-toggle").checked; var bake_lighting = document.getElementById("bake-lighting-toggle").checked; + var pipeline_merge = document.getElementById("pipeline-merge-toggle").checked; var scale = parseFloat(document.getElementById("scale-value-slider").value); + var cpuPercent = parseInt(document.getElementById("cpu-usage-slider").value, 10) || 90; // var ground_level = parseInt(document.getElementById("ground-level").value, 10); // DEPRECATED: Ground level input removed from UI var ground_level = -62; @@ -1610,7 +1645,9 @@ async function startGeneration() { spawnPoint: spawnPoint, telemetryConsent: telemetryConsent || false, worldFormat: getEffectiveWorldFormat(), - rotationAngle: rotationAngle + rotationAngle: rotationAngle, + cpuPercent: cpuPercent, + pipelineMerge: pipeline_merge }); console.log("Generation process started."); diff --git a/src/gui/locales/en-US.json b/src/gui/locales/en-US.json index f01a4c3f7..498c28110 100644 --- a/src/gui/locales/en-US.json +++ b/src/gui/locales/en-US.json @@ -9,6 +9,8 @@ "generation_process_started": "Generation process started.", "winter_mode": "Winter Mode", "world_scale": "World Scale", + "cpu_usage": "CPU Usage", + "pipeline_merge": "Faster merge (beta)", "custom_bounding_box": "Bounding Box", "floodfill_timeout": "Floodfill Timeout (sec)", "ground_level": "Ground Level", diff --git a/src/gui/locales/zh-CN.json b/src/gui/locales/zh-CN.json index 2c2b1627c..6b53b4da7 100644 --- a/src/gui/locales/zh-CN.json +++ b/src/gui/locales/zh-CN.json @@ -9,6 +9,8 @@ "generation_process_started": "生成过程已开始。", "winter_mode": "冬季模式", "world_scale": "世界比例", + "cpu_usage": "CPU 占用率", + "pipeline_merge": "加速合并（实验）", "custom_bounding_box": "Bounding Box", "floodfill_timeout": "填充超时（秒）", "ground_level": "地面高度", diff --git a/src/retrieve_data.rs b/src/retrieve_data.rs index 2c6156d22..0cd28d4b4 100644 --- a/src/retrieve_data.rs +++ b/src/retrieve_data.rs @@ -173,9 +173,9 @@ pub fn fetch_data_from_overpass( nwr["advertising"]; nwr["man_made"]; nwr["aeroway"]; + nwr["area:aeroway"]; nwr["3dmr"]; way["place"]["place"!~"^(ocean|sea|bay|strait|sound|fjord)$"]; - way; )->.relsinbbox; ( way(r.relsinbbox);