Parallelization

examples/CMakeLists.txt

@@ -3,5 +3,6 @@
 # SPDX-License-Identifier: MIT
 
 add_subdirectory(poisson)
+add_subdirectory(poisson_parallel)
 add_subdirectory(linelast)
 add_subdirectory(stokes)

examples/poisson_parallel/CMakeLists.txt

@@ -0,0 +1,15 @@
+# Copyright 2024 Lawrence Livermore National Security, LLC. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: MIT
+
+file(COPY poisson.sampling.yml DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel)
+file(COPY array.8.yml DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel)
+
+file(COPY generate_configs.py DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel)
+file(COPY setup_poisson.sh DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel)
+
+file(COPY ../stokes/meshes/square.o3.mesh DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel/meshes)
+file(COPY ../stokes/meshes/square-circle.msh.mfem DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel/meshes)
+file(COPY ../stokes/meshes/square-triangle.msh.mfem DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel/meshes)
+file(COPY ../stokes/meshes/square-star.msh.mfem DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel/meshes)
+file(COPY ../stokes/meshes/square-square.msh.mfem DESTINATION ${CMAKE_BINARY_DIR}/examples/poisson_parallel/meshes)

examples/poisson_parallel/array.8.yml

@@ -0,0 +1,92 @@
+main:
+#mode: run_example/sample_generation/build_rom/single_run
+  mode: single_run
+  use_rom: true
+  solver: poisson
+
+mesh:
+  type: component-wise
+  uniform_refinement: 0
+  component-wise:
+# you can try other config files you generated by specifying it here.
+    global_config: "configs/test.box-channel.8x8.h5"
+    components:
+      - name: "empty"
+        file: "meshes/square.o3.mesh"
+      - name: "square-circle"
+        file: "meshes/square-circle.msh.mfem"
+      - name: "square-square"
+        file: "meshes/square-square.msh.mfem"
+      - name: "square-triangle"
+        file: "meshes/square-triangle.msh.mfem"
+      - name: "square-star"
+        file: "meshes/square-star.msh.mfem"
+
+domain-decomposition:
+  type: interior_penalty
+
+discretization:
+  order: 2
+  full-discrete-galerkin: false
+
+solver:
+  direct_solve: true
+  use_amg: true
+  max_iter: 1000000
+  print_level: 0
+  absolute_tolerance: 1.0e-10
+  relative_tolerance: 1.0e-10
+
+visualization:
+  enabled: true
+  visualize_error: true
+  unified_paraview: false
+  file_path:
+    prefix: paraview/8x8/poisson_array_output
+
+parameterized_problem:
+  name: poisson_component
+
+single_run:
+  choose_from_random_sample: false
+  poisson_component:
+    k0: 0.6
+    k1: -0.6
+    bdr_k0: 0.05
+    bdr_k1: 0.04
+    offset: 0.1
+    bdr_offset: 0.
+
+basis:
+  prefix: "basis/poisson_basis"
+  number_of_basis: 10
+  tags:
+    - name: "empty"
+    - name: "square-circle"
+    - name: "square-square"
+    - name: "square-triangle"
+    - name: "square-star"
+  svd:
+    save_spectrum: true
+    update_right_sv: false
+  visualization:
+    enabled: false
+    prefix: poisson_comp
+
+model_reduction:
+  rom_handler_type: mfem
+  linear_solver_type: direct
+  linear_system_type: spd
+  hypre_matrix: yes
+# This is the input for saving ROM matrix/rhs system.
+  save_linear_system:
+    enabled: true
+    prefix: rom-system/poisson.8x8
+  preconditioner: gs
+  # individual/universal
+  subdomain_training: universal
+  save_operator:
+    level: component
+    prefix: "poisson_comp"
+  compare_solution:
+    enabled: true

examples/poisson_parallel/generate_configs.py

@@ -0,0 +1,24 @@
+import sys
+# Add path to root utils/python/ directory to find config scripts
+sys.path.insert(0, "../../../utils/python/")
+
+from config import Empty, ObjectInSpace
+from box_channel_config import BoxChannelConfig
+
+if __name__ == "__main__":
+    comp_list = {'empty': Empty(),
+                 'circle': ObjectInSpace('square-circle'),
+                 'square': ObjectInSpace('square-square'),
+                 'triangle': ObjectInSpace('square-triangle'),
+                 'star': ObjectInSpace('square-star'),}
+
+    example = BoxChannelConfig(2,2)
+    for name, comp in comp_list.items():
+        example.addComponent(comp)
+
+    example.GenerateAllConfigs(0)
+
+    test = BoxChannelConfig(8,8)
+    for name, comp in comp_list.items():
+        test.addComponent(comp)
+    test.CreateRandomConfig('test.box-channel.8x8.h5')

examples/poisson_parallel/poisson.sampling.yml

@@ -0,0 +1,115 @@
+main:
+  mode: sample_generation
+  use_rom: false
+  solver: poisson
+
+mesh:
+  uniform_refinement: 0
+  type: component-wise
+  component-wise:
+    global_config: "configs/box-channel.comp.h5"
+    components:
+      - name: "empty"
+        file: "meshes/square.o3.mesh"
+      - name: "square-circle"
+        file: "meshes/square-circle.msh.mfem"
+      - name: "square-square"
+        file: "meshes/square-square.msh.mfem"
+      - name: "square-triangle"
+        file: "meshes/square-triangle.msh.mfem"
+      - name: "square-star"
+        file: "meshes/square-star.msh.mfem"
+
+domain-decomposition:
+  type: interior_penalty
+
+discretization:
+  order: 2
+  full-discrete-galerkin: false
+
+solver:
+  direct_solve: true
+  use_amg: true # used only for iterative solver
+
+visualization:
+  enabled: false
+  unified_paraview: false
+  file_path:
+    prefix: poisson_output
+
+parameterized_problem:
+  name: poisson_component
+
+single_run:
+  poisson_component:
+    k0: 0.6
+    k1: -0.6
+    bdr_k0: 0.05
+    bdr_k1: 0.04
+    offset: 0.1
+    bdr_offset: 0.
+
+sample_generation:
+  maximum_number_of_snapshots: 4000
+  type: "random"
+  random_sample_generator:
+    number_of_samples: 1400
+  report_frequency: 10
+  file_path:
+    prefix: "./snapshots/poisson_array"
+  parameters:
+    - key: mesh/component-wise/global_config
+      type: filename
+      minimum: 0
+      maximum: 624
+      format: "./configs/samples/box-channel.config-%05d.h5"
+    - key: single_run/poisson_component/k0
+      type: double
+      minimum: -0.5
+      maximum: 0.5
+    - key: single_run/poisson_component/k1
+      type: double
+      minimum: -0.5
+      maximum: 0.5
+    - key: single_run/poisson_component/offset
+      type: double
+      minimum: 0.0
+      maximum: 1.0
+    - key: single_run/poisson_component/bdr_offset
+      type: double
+      minimum: 0.0
+      maximum: 1.0
+
+basis:
+  prefix: "basis/poisson_basis"
+  number_of_basis: 10
+  tags:
+    - name: empty
+      snapshot_files: ["./snapshots/poisson_array_sample_empty_snapshot"]
+    - name: square-circle
+      snapshot_files: ["./snapshots/poisson_array_sample_square-circle_snapshot"]
+    - name: square-square
+      snapshot_files: ["./snapshots/poisson_array_sample_square-square_snapshot"]
+    - name: square-star
+      snapshot_files: ["./snapshots/poisson_array_sample_square-star_snapshot"]
+    - name: square-triangle
+      snapshot_files: ["./snapshots/poisson_array_sample_square-triangle_snapshot"]
+  svd:
+    save_spectrum: true
+    update_right_sv: false
+  visualization:
+    enabled: false
+    prefix: poisson_comp
+
+model_reduction:
+  separate_variable_basis: false # true for vel/pres separate basis
+  rom_handler_type: mfem
+  linear_solver_type: direct
+  linear_system_type: spd
+  # individual/universal
+  subdomain_training: universal
+  save_operator:
+    level: component
+    prefix: "poisson_comp"
+  compare_solution:
+    enabled: false

examples/poisson_parallel/setup_poisson.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Generates input meshes for the Poisson example
+# Note: assumes this is run from the build/examples/poisson_parallel directory
+comp_script="../../../utils/python/box_comp_config.py"
+if [[ ! -f ${comp_script} ]]; then
+    echo "Could not find box_comp_config.py script"
+    exit 1
+fi
+
+# Generate box-channel.comp.h5
+python3 ${comp_script}
+
+# Generate all sample config meshes
+python3 generate_configs.py
+
+mkdir configs/
+mv box-channel.comp.h5 configs/
+mv test.box-channel.8x8.h5 configs/
+
+mkdir configs/samples/
+mv *.h5 configs/samples/

examples/stokes/array.8.yml

@@ -101,6 +101,7 @@ model_reduction:
   rom_handler_type: mfem
   linear_solver_type: direct
   linear_system_type: spd
+  hypre_matrix: yes
 # This is the input for saving ROM matrix/rhs system.
   save_linear_system:
     enabled: true

include/component_topology_handler.hpp

@@ -146,6 +146,10 @@ class ComponentTopologyHandler : public TopologyHandler
    virtual Array<InterfaceInfo>* const GetRefInterfaceInfos(const int &k) { return ref_interfaces[k]; }
    virtual Array<int>* GetBdrAttrComponentToGlobalMap(const int &m) { return bdr_c2g[m]; }
 
+   int LocalSubdomainIndex(int global_subdomain) override;
+   int GlobalSubdomainIndex(int local_subdomain) override;
+   int GlobalSubdomainRank(int global_subdomain) override;
+
    // return component indexes for a reference port (ComponentTopologyHandler only)
    virtual void GetComponentPair(const int &ref_port_idx, int &comp1, int &comp2);
    virtual void GetRefPortInfo(const int &ref_port_idx, int &comp1, int &comp2, int &attr1, int &attr2);
@@ -182,6 +186,11 @@ class ComponentTopologyHandler : public TopologyHandler
    void SetupPorts();
 
    bool ComponentBdrAttrCheck(Mesh *comp);
+
+private:
+   Mesh* CreateMesh(int global_subdomain) const;
+   void SetupPortNeighborMeshes(const std::string &global_config);
+   void SetupBoundaryAttributes();
 };
 
 #endif

include/interface_form.hpp

@@ -19,6 +19,7 @@ class InterfaceForm
    mutable TimeProfiler timer;
 
    int numSub = -1;
+   int numSubStored = -1;
    int skip_zeros = 1;
 
    Array<Mesh *> meshes;                  // not owned
@@ -95,6 +96,7 @@ class MixedInterfaceForm
 {
 protected:
    int numSub = -1;
+   int numSubStored = -1;
    int skip_zeros = 1;
 
    Array<Mesh *> meshes;   // not owned

include/multiblock_solver.hpp

@@ -44,11 +44,18 @@ friend class ParameterizedProblem;
    Mesh *pmesh = NULL;  // parent mesh. only available from SubMeshTopologyHandler.
    Array<Mesh*> meshes;
 
-   // Informations received from Topology Handler.
+   // Information received from Topology Handler.
    int numSub;   // number of subdomains.
+   int numSubLoc;   // number of local subdomains.
+   int numSubStored;   // number of subdomains stored on this MPI rank.
    int dim;      // Spatial dimension.
    Array<int> global_bdr_attributes;   // boundary attributes of global system.
 
+   // MPI partitioning data
+   Array<int> localNumBlocks;
+   std::array<int,2> localBlocks;
+   Array<int> neighbors;
+
    // Solution dimension, by default -1 (scalar).
    int udim = -1;       // vector dimension of the entire solution variable
    int num_var = -1;    // number of variables

include/poisson_solver.hpp

@@ -33,6 +33,7 @@ friend class ParameterizedProblem;
    HYPRE_BigInt sys_glob_size;
    HYPRE_BigInt sys_row_starts[2];
    HypreParMatrix *globalMat_hypre = NULL;
+   HypreParMatrix *systemOp_hypre = NULL;
    MUMPSSolver *mumps = NULL;
 
    // operators
@@ -102,6 +103,19 @@ friend class ParameterizedProblem;
 
 protected:
    virtual void SetMUMPSSolver();
+   void SetupMUMPSSolverSerial();
+   void SetupMUMPSSolverParallel();
+
+   void CreateHypreParMatrix();
+   void SetGlobalOffsets();
+
+   // Data for HypreParMatrix construction
+   Array<int> global_offsets;
+   Array<int> global_var_offsets;
+   SparseMatrix *hdiag = NULL;
+   SparseMatrix *hoffd = NULL;
+   HYPRE_BigInt *cmap = NULL;
+
 };
 
 #endif