Skip to content

Commit

Permalink
[Draft] Fix: MPI-Distribution w/o SC
Browse files Browse the repository at this point in the history 
Without space charge, we can simply split the particles up during
init in equal chunks and leave them there forever.

This fixes grid generation by setting the `max_grid_size` to the
AMReX `blocking_factor` (if not set by the user). Also, this
avoids all calls to `ResizeMesh()` and `Redistribute` for now.
  • Loading branch information
@ax3l
ax3l committed Jan 10, 2024
1 parent f907561 commit e326afb
Show file tree
Hide file tree
Showing 4 changed files with 50 additions and 11 deletions.
27 changes: 23 additions & 4 deletions src/ImpactX.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,14 @@ namespace impactx
{
BL_PROFILE("ImpactX::initGrids");

// In init_amr_core, when
// n_cells has been set using temporary values earlier. We now know the true value of
// n_cells, so we recompute the Geometry objects for each level here *if* the user
// has set n_cells in the inputs file
{
amrex::Vector<int> n_cell(AMREX_SPACEDIM);
amrex::ParmParse const pp_amr("amr");
pp_amr.queryarr("n_cell", n_cell);
pp_amr.getarr("n_cell", n_cell);

amrex::IntVect const lo(amrex::IntVect::TheZeroVector());
amrex::IntVect hi(n_cell);
Expand Down Expand Up @@ -176,13 +177,18 @@ namespace impactx
for (int slice_step = 0; slice_step < nslice; ++slice_step) {
BL_PROFILE("ImpactX::evolve::slice_step");
global_step++;
auto const global_np = m_particle_container->TotalNumberOfParticles(false, false);
auto const local_np = m_particle_container->TotalNumberOfParticles(false, true);

amrex::Print() << " ++++ Starting global_step=" << global_step
<< " slice_step=" << slice_step << "\n";

// Space-charge calculation: turn off if there is only 1 particle
if (space_charge &&
m_particle_container->TotalNumberOfParticles(false, false) > 1) {
amrex::AllPrint() << "Local np=" << local_np << " out of global np=" << global_np
<< " at Proc=" << amrex::ParallelDescriptor::MyProc() << "\n";

// Space-charge calculation: turn off if there is only 1 particle
if (space_charge && global_np > 1)
{
// transform from x',y',t to x,y,z
transformation::CoordinateTransformation(
*m_particle_container,
Expand Down Expand Up @@ -220,6 +226,19 @@ namespace impactx
transformation::CoordinateTransformation(*m_particle_container,
CoordSystem::s);
}
// balance particle beam in sims w/o space charge
/*else if (global_np > 1)
{
// Resize the mesh, based on `m_particle_container` extent
amrex::ParmParse pp_geometry("geometry");
amrex::Vector<amrex::Real> prob_relative(max_level + 1, 1.0);
pp_geometry.addarr("prob_relative", prob_relative);
ResizeMesh();
// Redistribute particles in the new mesh in x, y, t
m_particle_container->Redistribute();
}*/

// for later: original Impact implementation as an option
// Redistribute particles in x',y',t
Expand Down
22 changes: 18 additions & 4 deletions src/initialization/InitAmrCore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ namespace details

amrex::Vector<int> n_cell(AMREX_SPACEDIM);
bool const has_ncell = pp_amr.queryarr("n_cell", n_cell);
amrex::Print() << "has_ncell=" << has_ncell << "\n";
if (has_ncell)
return amrex_amrcore_gridding();
else
Expand All @@ -88,8 +89,9 @@ namespace details
{
amrex::ParmParse const pp_amr("amr");

// Domain index space - we use temporary values here, then fix later
amrex::Vector<int> const n_cell = {AMREX_D_DECL(256,256,256)};
// Domain index space
amrex::Vector<int> n_cell;
pp_amr.getarr("n_cell", n_cell);
amrex::Box const domain(amrex::IntVect(0), amrex::IntVect(n_cell));

// Domain physical size
Expand All @@ -111,14 +113,26 @@ namespace details
AmrCoreData
one_box_per_rank ()
{
// Domain index space
amrex::AmrInfo amr_info;

// set max_grid_size to blocking_factor to fix the number of boxes we generate
amrex::ParmParse pp_amr("amr");
bool const has_max_grid_size =
pp_amr.countRecords("max_grid_size") > 0 ||
pp_amr.countRecords("max_grid_size_x") > 0 ||
pp_amr.countRecords("max_grid_size_y") > 0 ||
pp_amr.countRecords("max_grid_size_z") > 0;
if (!has_max_grid_size) {
amrex::Vector<int> const bf_lvl0(amr_info.blocking_factor[0].begin(), amr_info.blocking_factor[0].end());
pp_amr.addarr("max_grid_size", bf_lvl0);
}

// Domain index space
const int nprocs = amrex::ParallelDescriptor::NProcs();
const amrex::IntVect high_end = amr_info.blocking_factor[0]
* amrex::IntVect(AMREX_D_DECL(nprocs,1,1)) - amrex::IntVect(1);
amrex::Box const domain(amrex::IntVect(0), high_end);
// adding amr.n_cell for consistency
amrex::ParmParse pp_amr("amr");
auto const n_cell_iv = domain.size();
amrex::Vector<int> const n_cell_v(n_cell_iv.begin(), n_cell_iv.end());
pp_amr.addarr("n_cell", n_cell_v);
Expand Down
4 changes: 2 additions & 2 deletions src/initialization/InitDistribution.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,8 @@ namespace impactx
// Resize the mesh to fit the spatial extent of the beam and then
// redistribute particles, so they reside on the MPI rank that is
// responsible for the respective spatial particle position.
this->ResizeMesh();
m_particle_container->Redistribute();
//this->ResizeMesh();
//m_particle_container->Redistribute();
}

void ImpactX::initBeamDistributionFromInputs ()
Expand Down
8 changes: 7 additions & 1 deletion src/initialization/InitMeshRefinement.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,13 @@ namespace detail
amrex::Vector<amrex::Real>
read_mr_prob_relative ()
{
amrex::ParmParse pp_algo("algo");
amrex::ParmParse pp_amr("amr");
amrex::ParmParse pp_geometry("geometry");

bool space_charge = false;
pp_algo.query("space_charge", space_charge);

int max_level = 0;
pp_amr.query("max_level", max_level);

Expand All @@ -43,7 +47,7 @@ namespace detail
prob_relative[0] = 3.0; // top/bottom pad the beam on the lowest level by default by its width
pp_geometry.queryarr("prob_relative", prob_relative);

if (prob_relative[0] < 3.0)
if (prob_relative[0] < 3.0 && space_charge)
ablastr::warn_manager::WMRecordWarning(
"ImpactX::read_mr_prob_relative",
"Dynamic resizing of the mesh uses a geometry.prob_relative "
Expand Down Expand Up @@ -315,6 +319,8 @@ namespace detail
g.ProbDomain(rb[lev]);
SetGeometry(lev, g);

amrex::AllPrint() << "lev=" << lev << " Geometry=" << g << "\n";

m_particle_container->SetParticleGeometry(lev, g);
}
}
Expand Down

0 comments on commit e326afb

Please sign in to comment.