diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml index b15b8091b..09ec68874 100644 --- a/.gitlab/build_toss4.yml +++ b/.gitlab/build_toss4.yml @@ -43,7 +43,7 @@ toss4-llvm_19_1_3-src: variables: COMPILER: "llvm@19.1.3" HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake" - EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug" + EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DSERAC_USE_DFEM=ON" # Only run integration tests on one spec (Disabled until integration tests are reenabled) # DO_INTEGRATION_TESTS: "yes" # ALLOC_NODES: "2" @@ -51,6 +51,17 @@ toss4-llvm_19_1_3-src: ALLOC_TIME: "30" extends: .src_build_on_toss4 +# TODO: debug why Enzyme isn't working with Tribol on codevelop +# toss4-llvm_19_1_3-src-codevelop: +# variables: +# COMPILER: "llvm@19.1.3" +# HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake" +# EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DSERAC_ENABLE_CODEVELOP=ON -DSERAC_USE_DFEM=ON" +# EXTRA_BUILD_OPTIONS: "--skip-install" +# ALLOC_NODES: "1" +# ALLOC_TIME: "30" +# extends: .src_build_on_toss4 + toss4-gcc_13_3_1-src: variables: COMPILER: "gcc@13.3.1" diff --git a/.gitlab/build_toss4_cray.yml b/.gitlab/build_toss4_cray.yml index a3816b897..68b84c3ac 100644 --- a/.gitlab/build_toss4_cray.yml +++ b/.gitlab/build_toss4_cray.yml @@ -33,23 +33,24 @@ # Build jobs toss4_cray-rocmcc_6_2_1-src: - variables: - COMPILER: "rocmcc@6.2.1" - HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" - EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug" - ALLOC_NODES: "1" - ALLOC_TIME: "30" - extends: .src_build_on_toss4_cray + variables: + COMPILER: "rocmcc@6.2.1" + HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" + EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug" + ALLOC_NODES: "1" + ALLOC_TIME: "30" + extends: .src_build_on_toss4_cray toss4_cray-rocmcc_6_2_1-src-codevelop: - variables: - COMPILER: "rocmcc@6.2.1" - HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" - EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DSMITH_ENABLE_CODEVELOP=ON" - EXTRA_BUILD_OPTIONS: "--skip-install" - ALLOC_NODES: "1" - ALLOC_TIME: "30" - extends: .src_build_on_toss4_cray + variables: + COMPILER: "rocmcc@6.2.1" + HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" + # TODO: Add -DSERAC_USE_DFEM=ON when TPLs are updated + EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DSMITH_ENABLE_CODEVELOP=ON" + EXTRA_BUILD_OPTIONS: "--skip-install" + ALLOC_NODES: "1" + ALLOC_TIME: "30" + extends: .src_build_on_toss4_cray # NOTE: SPEC should matches specs.json, but devtools and profiling variants removed toss4_cray-rocmcc_6_2_1-tpl: diff --git a/cmake/smith-config.cmake.in b/cmake/smith-config.cmake.in index a4091685b..bf65912ca 100644 --- a/cmake/smith-config.cmake.in +++ b/cmake/smith-config.cmake.in @@ -38,6 +38,7 @@ if(NOT SMITH_FOUND) set(SMITH_USE_CALIPER @SMITH_USE_CALIPER@) set(SMITH_USE_CONDUIT @SMITH_USE_CONDUIT@) set(SMITH_USE_CONTINUATION @SMITH_USE_CONTINUATION@) + set(SMITH_USE_DFEM @SMITH_USE_DFEM@) set(SMITH_USE_ENZYME @SMITH_USE_ENZYME@) set(SMITH_USE_HDF5 @SMITH_USE_HDF5@) set(SMITH_USE_MFEM @SMITH_USE_MFEM@) diff --git a/cmake/thirdparty/SetupSmithThirdParty.cmake b/cmake/thirdparty/SetupSmithThirdParty.cmake index 7dc4c1874..54af4d367 100644 --- a/cmake/thirdparty/SetupSmithThirdParty.cmake +++ b/cmake/thirdparty/SetupSmithThirdParty.cmake @@ -197,6 +197,40 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND) endif() message(STATUS "ARPACK support is ${ARPACK_FOUND}") + #------------------------------------------------------------------------------ + # Enzyme + #------------------------------------------------------------------------------ + if (ENZYME_DIR) + smith_assert_is_directory(DIR_VARIABLE ENZYME_DIR) + set(Enzyme_ROOT ${ENZYME_DIR} CACHE PATH "") + find_dependency(Enzyme REQUIRED) + + smith_assert_find_succeeded(PROJECT_NAME Enzyme + TARGET ClangEnzymeFlags + DIR_VARIABLE ENZYME_DIR) + + message(STATUS "Checking for Target 'ClangEnzymeFlags' plugin target exists..") + get_target_property(_clangenzyme_opts ClangEnzymeFlags INTERFACE_COMPILE_OPTIONS) + if("${_clangenzyme_opts}" MATCHES "\\$]+)>") + set(_enzyme_target "${CMAKE_MATCH_1}") + + # Check if the extracted target exists + if(TARGET "${_enzyme_target}") + message(STATUS "Found 'ClangEnzymeFlags' plugin target: ${_enzyme_target}") + else() + message(FATAL_ERROR "'ClangEnzymeFlags' plugin target '${_enzyme_target}' referenced in INTERFACE_COMPILE_OPTIONS does not exist.") + endif() + else() + message(STATUS "Skipped check. `ClangEnzymeFlags` target does not reference another target") + endif() + + message(STATUS "Enzyme support is ON") + set(ENZYME_FOUND TRUE) + else() + message(STATUS "Enzyme support is OFF") + set(ENZYME_FOUND FALSE) + endif() + #------------------------------------------------------------------------------ # MFEM #------------------------------------------------------------------------------ @@ -294,6 +328,12 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND) endif() set(MFEM_USE_UMPIRE OFF CACHE BOOL "") set(MFEM_USE_ZLIB ON CACHE BOOL "") + if(ENZYME_DIR) + smith_assert_is_directory(DIR_VARIABLE ENZYME_DIR) + set(MFEM_USE_ENZYME ON CACHE BOOL "") + else() + set(MFEM_USE_ENZYME OFF CACHE BOOL "") + endif() #### MFEM Configuration Options @@ -508,40 +548,6 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND) set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE) endif() - #------------------------------------------------------------------------------ - # Enzyme - #------------------------------------------------------------------------------ - if (ENZYME_DIR) - smith_assert_is_directory(DIR_VARIABLE ENZYME_DIR) - set(Enzyme_ROOT ${ENZYME_DIR} CACHE PATH "") - find_dependency(Enzyme REQUIRED) - - smith_assert_find_succeeded(PROJECT_NAME Enzyme - TARGET ClangEnzymeFlags - DIR_VARIABLE ENZYME_DIR) - - message(STATUS "Checking for Target 'ClangEnzymeFlags' plugin target exists..") - get_target_property(_clangenzyme_opts ClangEnzymeFlags INTERFACE_COMPILE_OPTIONS) - if("${_clangenzyme_opts}" MATCHES "\\$]+)>") - set(_enzyme_target "${CMAKE_MATCH_1}") - - # Check if the extracted target exists - if(TARGET "${_enzyme_target}") - message(STATUS "Found 'ClangEnzymeFlags' plugin target: ${_enzyme_target}") - else() - message(FATAL_ERROR "'ClangEnzymeFlags' plugin target '${_enzyme_target}' referenced in INTERFACE_COMPILE_OPTIONS does not exist.") - endif() - else() - message(STATUS "Skipped check. `ClangEnzymeFlags` target does not reference another target") - endif() - - message(STATUS "Enzyme support is ON") - set(ENZYME_FOUND TRUE) - else() - message(STATUS "Enzyme support is OFF") - set(ENZYME_FOUND FALSE) - endif() - #------------------------------------------------------------------------------ # Tribol #------------------------------------------------------------------------------ @@ -593,7 +599,7 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND) $ ) - set(TRIBOL_FOUND TRUE CACHE BOOL "" FORCE) + set(TRIBOL_FOUND ON CACHE BOOL "" FORCE) set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE) endif() diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2c874419a..b98f72a5c 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -29,6 +29,9 @@ add_subdirectory(contact) # Add the buckling examples add_subdirectory(buckling) +# Add the explicit dynamics examples +add_subdirectory(explicit) + # Add the uniaxial examples add_subdirectory(uniaxial) diff --git a/examples/explicit/CMakeLists.txt b/examples/explicit/CMakeLists.txt new file mode 100644 index 000000000..d94155715 --- /dev/null +++ b/examples/explicit/CMakeLists.txt @@ -0,0 +1,34 @@ +# Copyright (c) Lawrence Livermore National Security, LLC and +# other Serac Project Developers. See the top-level LICENSE file for +# details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +if(SERAC_USE_DFEM) + set(EXPLICIT_DYNAMICS_EXAMPLES_SOURCES + dfem_gpu.cpp + dfem_force_gpu.cpp + ) + + foreach(filename ${EXPLICIT_DYNAMICS_EXAMPLES_SOURCES}) + get_filename_component(example_name ${filename} NAME_WE) + + blt_add_executable(NAME explicit_${example_name} + SOURCES ${filename} + OUTPUT_DIR ${EXAMPLE_OUTPUT_DIRECTORY} + DEPENDS_ON serac_physics serac_mesh_utils) + + serac_add_example_test(NAME explicit_${example_name} + COMMAND explicit_${example_name} + NUM_MPI_TASKS 4) + + endforeach() + + install( + FILES + ${EXPLICIT_DYNAMICS_EXAMPLES_SOURCES} + DESTINATION + examples/serac/explicit + ) + +endif() diff --git a/examples/explicit/dfem_force_gpu.cpp b/examples/explicit/dfem_force_gpu.cpp new file mode 100644 index 000000000..1bd91abd1 --- /dev/null +++ b/examples/explicit/dfem_force_gpu.cpp @@ -0,0 +1,217 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Serac Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +#include "mfem.hpp" +#include "serac/infrastructure/accelerator.hpp" +#include "serac/infrastructure/application_manager.hpp" +#include "serac/physics/boundary_conditions/boundary_condition_manager.hpp" +#include "serac/physics/mesh.hpp" +#include "serac/physics/state/state_manager.hpp" + +#include "serac/physics/functional_weak_form.hpp" +#include "serac/physics/dfem_solid_weak_form.hpp" +#include "serac/physics/dfem_mass_weak_form.hpp" + +auto element_shape = mfem::Element::QUADRILATERAL; + +const std::string MESHTAG = "mesh"; + +namespace mfem { +namespace future { + +/** + * @brief Compute Green's strain from the displacement gradient + */ +template +SERAC_HOST_DEVICE auto greenStrain(const tensor& grad_u) +{ + return 0.5 * (grad_u + transpose(grad_u) + dot(transpose(grad_u), grad_u)); +} + +} // namespace future +} // namespace mfem + +namespace serac { + +// NOTE (EBC): NeoHookean is not working with dfem on device with HIP, since some needed LLVM intrinsics are not +// implemented in Enzyme with the call to log1p()/log(). +struct StVenantKirchhoffWithFieldDensityDfem { + static constexpr int dim = 2; + + /** + * @brief stress calculation for a St. Venant Kirchhoff material model + * + * @tparam T Type of the displacement gradient components (number-like) + * + * @param[in] grad_u Displacement gradient + * + * @return The first Piola stress + */ + template + SERAC_HOST_DEVICE auto pkStress(double, const mfem::future::tensor& du_dX, + const mfem::future::tensor&, const Density&) const + { + auto I = mfem::future::IdentityMatrix(); + auto F = du_dX + I; + const auto E = mfem::future::greenStrain(du_dX); + + // stress + const auto S = K * mfem::future::tr(E) * I + 2.0 * G * mfem::future::dev(E); + return mfem::future::tuple{mfem::future::dot(F, S)}; + } + + /// @brief interpolates density field + template + SERAC_HOST_DEVICE auto density(const Density& density) const + { + return density; + } + + double K; ///< Bulk modulus + double G; ///< Shear modulus +}; + +} // namespace serac + +int main(int argc, char* argv[]) +{ + static constexpr int dim = 2; + static constexpr int disp_order = 1; + + using SolidMaterialDfem = serac::StVenantKirchhoffWithFieldDensityDfem; + + enum STATE + { + DISPLACEMENT, + VELOCITY, + ACCELERATION, + COORDINATES + }; + + enum PARAMS + { + DENSITY + }; + + // Command line-modifiable variables + int n_els = 8; + bool use_gpu = false; + bool write_output = false; + + // Handle command line arguments + axom::CLI::App app{"Explicit dynamics"}; + // Mesh options + app.add_option("--nels", n_els, "Number of elements in the x and y directions")->check(axom::CLI::PositiveNumber); + // GPU options + app.add_flag("--gpu,!--no-gpu", use_gpu, "Execute on GPU (where available)"); + // Output options + app.add_flag("--output,!--no-output", write_output, "Save output to disk (e.g. for debugging)"); + + // Need to allow extra arguments for PETSc support + app.set_help_flag("--help"); + app.allow_extras()->parse(argc, argv); + + auto exec_space = use_gpu ? serac::ExecutionSpace::GPU : serac::ExecutionSpace::CPU; + + serac::ApplicationManager applicationManager(argc, argv, MPI_COMM_WORLD, true, exec_space); + + MPI_Barrier(MPI_COMM_WORLD); + + axom::sidre::DataStore datastore; + serac::StateManager::initialize(datastore, "solid_dynamics"); + + // create mesh + constexpr double length = 1.0; + constexpr double width = 1.0; + int nel_x = n_els; + int nel_y = n_els; + auto mesh = std::make_shared( + mfem::Mesh::MakeCartesian2D(nel_x, nel_y, element_shape, true, length, width), MESHTAG, 0, 0); + + // create residual evaluator + using VectorSpace = serac::H1; + using DensitySpace = serac::L2; + serac::FiniteElementState disp = serac::StateManager::newState(VectorSpace{}, "displacement", mesh->tag()); + serac::FiniteElementState velo = serac::StateManager::newState(VectorSpace{}, "velocity", mesh->tag()); + serac::FiniteElementState accel = serac::StateManager::newState(VectorSpace{}, "acceleration", mesh->tag()); + serac::FiniteElementState coords = serac::StateManager::newState(VectorSpace{}, "coordinates", mesh->tag()); + serac::FiniteElementState density = serac::StateManager::newState(DensitySpace{}, "density", mesh->tag()); + + std::vector states{disp, velo, accel, coords}; + std::vector params{density}; + + std::string physics_name = "solid"; + double E = 1.0e3; + double nu = 0.3; + + using SolidT = serac::DfemSolidWeakForm; + auto solid_dfem_weak_form = + std::make_shared(physics_name, mesh, states[DISPLACEMENT].space(), getSpaces(params)); + + SolidMaterialDfem dfem_mat; + dfem_mat.K = E / (3.0 * (1.0 - 2.0 * nu)); // bulk modulus + dfem_mat.G = E / (2.0 * (1.0 + nu)); // shear modulus + int ir_order = 3; + const mfem::IntegrationRule& displacement_ir = mfem::IntRules.Get(disp.space().GetFE(0)->GetGeomType(), ir_order); + mfem::Array solid_attrib({1}); + solid_dfem_weak_form->setMaterial>(solid_attrib, dfem_mat, + displacement_ir); + + mfem::future::tensor g({0.0, -9.81}); // gravity vector + mfem::future::tuple, mfem::future::Value, + mfem::future::Value, mfem::future::Gradient, + mfem::future::Weight, mfem::future::Value> + g_inputs{}; + mfem::future::tuple> g_outputs{}; + solid_dfem_weak_form->addBodyIntegral( + solid_attrib, + [=] SERAC_HOST_DEVICE(const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor& dX_dxi, mfem::real_t weight, double) { + auto J = mfem::future::det(dX_dxi) * weight; + return mfem::future::tuple{g * J}; + }, + g_inputs, g_outputs, displacement_ir, std::index_sequence<>{}); + + states[DISPLACEMENT] = 0.0; + states[VELOCITY].setFromFieldFunction([](serac::tensor) { + serac::tensor u({0.0, -1.0}); + return u; + }); + states[ACCELERATION] = 0.0; + states[COORDINATES].setFromGridFunction(static_cast(*mesh->mfemParMesh().GetNodes())); + params[DENSITY] = 1.0; + + double time = 0.0; + constexpr double dt = 0.0001; + constexpr size_t num_steps = 5000; + + const auto& u = states[DISPLACEMENT]; + const auto& v = states[VELOCITY]; + const auto& a = states[ACCELERATION]; + + auto v_pred = v; + v_pred.Add(0.5 * dt, a); + auto u_pred = u; + u_pred.Add(dt, v_pred); + + std::vector pred_states = {&u_pred, &v_pred, &a, &states[COORDINATES], ¶ms[DENSITY]}; + + axom::utilities::Timer timer(true); + for (size_t step = 0; step < num_steps; ++step) { + auto no_mass_resid = solid_dfem_weak_form->residual(time, dt, &u_pred, pred_states); + time += dt; + } + timer.stop(); + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) { + std::cout << "Total time: " << timer.elapsedTimeInMilliSec() << " milliseconds" << std::endl; + } + + return 0; +} diff --git a/examples/explicit/dfem_gpu.cpp b/examples/explicit/dfem_gpu.cpp new file mode 100644 index 000000000..ad8e9b65f --- /dev/null +++ b/examples/explicit/dfem_gpu.cpp @@ -0,0 +1,240 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Serac Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +#include "mfem.hpp" +#include "serac/infrastructure/accelerator.hpp" +#include "serac/infrastructure/application_manager.hpp" +#include "serac/physics/boundary_conditions/boundary_condition_manager.hpp" +#include "serac/physics/mesh.hpp" +#include "serac/physics/state/state_manager.hpp" + +#include "serac/physics/functional_weak_form.hpp" +#include "serac/physics/dfem_solid_weak_form.hpp" +#include "serac/physics/dfem_mass_weak_form.hpp" + +auto element_shape = mfem::Element::QUADRILATERAL; + +const std::string MESHTAG = "mesh"; + +namespace mfem { +namespace future { + +/** + * @brief Compute Green's strain from the displacement gradient + */ +template +SERAC_HOST_DEVICE auto greenStrain(const tensor& grad_u) +{ + return 0.5 * (grad_u + transpose(grad_u) + dot(transpose(grad_u), grad_u)); +} + +} // namespace future +} // namespace mfem + +namespace serac { + +// NOTE (EBC): NeoHookean is not working with dfem on device with HIP, since some needed LLVM intrinsics are not +// implemented in Enzyme with the call to log1p()/log(). +struct StVenantKirchhoffWithFieldDensityDfem { + static constexpr int dim = 2; + + /** + * @brief stress calculation for a St. Venant Kirchhoff material model + * + * @tparam T Type of the displacement gradient components (number-like) + * + * @param[in] grad_u Displacement gradient + * + * @return The first Piola stress + */ + template + SERAC_HOST_DEVICE auto pkStress(double, const mfem::future::tensor& du_dX, + const mfem::future::tensor&, const Density&) const + { + auto I = mfem::future::IdentityMatrix(); + auto F = du_dX + I; + const auto E = mfem::future::greenStrain(du_dX); + + // stress + const auto S = K * mfem::future::tr(E) * I + 2.0 * G * mfem::future::dev(E); + return mfem::future::tuple{mfem::future::dot(F, S)}; + } + + /// @brief interpolates density field + template + SERAC_HOST_DEVICE auto density(const Density& density) const + { + return density; + } + + double K; ///< Bulk modulus + double G; ///< Shear modulus +}; + +} // namespace serac + +int main(int argc, char* argv[]) +{ + static constexpr int dim = 2; + static constexpr int disp_order = 1; + + using SolidMaterialDfem = serac::StVenantKirchhoffWithFieldDensityDfem; + + enum STATE + { + DISPLACEMENT, + VELOCITY, + ACCELERATION, + COORDINATES + }; + + enum PARAMS + { + DENSITY + }; + + // Command line-modifiable variables + int n_els = 8; + bool use_gpu = false; + bool write_output = false; + + // Handle command line arguments + axom::CLI::App app{"Explicit dynamics"}; + // Mesh options + app.add_option("--nels", n_els, "Number of elements in the x and y directions")->check(axom::CLI::PositiveNumber); + // GPU options + app.add_flag("--gpu,!--no-gpu", use_gpu, "Execute on GPU (where available)"); + // Output options + app.add_flag("--output,!--no-output", write_output, "Save output to disk (e.g. for debugging)"); + + // Need to allow extra arguments for PETSc support + app.set_help_flag("--help"); + app.allow_extras()->parse(argc, argv); + + auto exec_space = use_gpu ? serac::ExecutionSpace::GPU : serac::ExecutionSpace::CPU; + + serac::ApplicationManager applicationManager(argc, argv, MPI_COMM_WORLD, true, exec_space); + + MPI_Barrier(MPI_COMM_WORLD); + + axom::sidre::DataStore datastore; + serac::StateManager::initialize(datastore, "solid_dynamics"); + + // create mesh + constexpr double length = 1.0; + constexpr double width = 1.0; + int nel_x = n_els; + int nel_y = n_els; + auto mesh = std::make_shared( + mfem::Mesh::MakeCartesian2D(nel_x, nel_y, element_shape, true, length, width), MESHTAG, 0, 0); + + // create residual evaluator + using VectorSpace = serac::H1; + using DensitySpace = serac::L2; + serac::FiniteElementState disp = serac::StateManager::newState(VectorSpace{}, "displacement", mesh->tag()); + serac::FiniteElementState velo = serac::StateManager::newState(VectorSpace{}, "velocity", mesh->tag()); + serac::FiniteElementState accel = serac::StateManager::newState(VectorSpace{}, "acceleration", mesh->tag()); + serac::FiniteElementState density = serac::StateManager::newState(DensitySpace{}, "density", mesh->tag()); + + std::vector states{disp, velo, accel}; + std::vector params{density}; + + std::string physics_name = "solid"; + double E = 1.0e3; + double nu = 0.3; + + using SolidT = serac::DfemSolidWeakForm; + auto solid_dfem_weak_form = + std::make_shared(physics_name, mesh, states[DISPLACEMENT].space(), getSpaces(params)); + + SolidMaterialDfem dfem_mat; + dfem_mat.K = E / (3.0 * (1.0 - 2.0 * nu)); // bulk modulus + dfem_mat.G = E / (2.0 * (1.0 + nu)); // shear modulus + int ir_order = 3; + const mfem::IntegrationRule& displacement_ir = mfem::IntRules.Get(disp.space().GetFE(0)->GetGeomType(), ir_order); + mfem::Array solid_attrib({1}); + solid_dfem_weak_form->setMaterial>(solid_attrib, dfem_mat, + displacement_ir); + + mfem::future::tensor g({0.0, -9.81}); // gravity vector + mfem::future::tuple, mfem::future::Value, + mfem::future::Value, mfem::future::Gradient, + mfem::future::Weight, mfem::future::Value> + g_inputs{}; + mfem::future::tuple> g_outputs{}; + solid_dfem_weak_form->addBodyIntegral( + solid_attrib, + [=] SERAC_HOST_DEVICE(const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor& dX_dxi, mfem::real_t weight, double) { + auto J = mfem::future::det(dX_dxi) * weight; + return mfem::future::tuple{g * J}; + }, + g_inputs, g_outputs, displacement_ir, std::index_sequence<>{}); + + auto bc_manager = std::make_shared(mesh->mfemParMesh()); + auto zero_coeff = std::make_shared(0.0); + bc_manager->addEssential({1}, zero_coeff, states[DISPLACEMENT].space()); + + states[DISPLACEMENT] = 0.0; + states[VELOCITY].setFromFieldFunction([](serac::tensor) { + serac::tensor u({0.0, -1.0}); + return u; + }); + states[ACCELERATION] = 0.0; + params[DENSITY] = 1.0; + + auto mass_dfem_weak_form = serac::create_solid_mass_weak_form(physics_name, mesh, states[DISPLACEMENT], + params[0], displacement_ir); + + // create time advancer + auto advancer = + std::make_shared(solid_dfem_weak_form, mass_dfem_weak_form, bc_manager); + + serac::FiniteElementState coords_state( + *static_cast(mesh->mfemParMesh().GetNodes())->ParFESpace(), "coordinates"); + coords_state.setFromGridFunction(*static_cast(mesh->mfemParMesh().GetNodes())); + + double time = 0.0; + constexpr double dt = 0.0001; + int cycle = 0; + constexpr size_t num_steps = 5000; + axom::utilities::Timer timer(true); + for (size_t step = 0; step < num_steps; ++step) { + if (write_output && cycle % 100 == 0) { + for (auto& state : states) { + // copy to grid function + serac::StateManager::updateState(state); + } + for (auto& param : params) { + // copy to grid function + serac::StateManager::updateState(param); + } + serac::StateManager::save(time, cycle, mesh->tag()); + } + ++cycle; + auto state_ptrs = serac::getConstFieldPointers(states); + state_ptrs.push_back(&coords_state); + auto new_states_and_time = advancer->advanceState(state_ptrs, getConstFieldPointers(params), time, dt); + time = std::get<1>(new_states_and_time); + for (size_t i = 0; i < states.size(); ++i) { + states[i] = std::get<0>(new_states_and_time)[i]; + } + } + timer.stop(); + // copy to host + states[DISPLACEMENT].HostRead(); + double max_disp = mfem::ParNormlp(states[DISPLACEMENT], 2, MPI_COMM_WORLD); + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (rank == 0) { + std::cout << "Max displacement: " << max_disp << std::endl; + std::cout << "Total time: " << timer.elapsedTimeInMilliSec() << " milliseconds" << std::endl; + } + + return 0; +} diff --git a/host-configs/dfem/ruby-toss_4_x86_64_ib-clang@19.1.7.cmake b/host-configs/dfem/ruby-toss_4_x86_64_ib-clang@19.1.7.cmake new file mode 100644 index 000000000..fa0cd4152 --- /dev/null +++ b/host-configs/dfem/ruby-toss_4_x86_64_ib-clang@19.1.7.cmake @@ -0,0 +1,138 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/caliper-2.12.1-5o3zfbeosigjwrp5ffizjs6sf7pyoqjv;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/libunwind-1.8.1-tgmphz65mn62bautt2ksl45ckecw5e2c;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/fmt-11.0.2-xpfumigjok66664dhruf6tfaa7gwcn3n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/netlib-scalapack-2.2.2-fuk7tt7il4e6tekxxctc625g4xk4hqeu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/tce;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/cppcheck-2.9;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/doxygen-1.9.8;/usr/tce/packages/clang/clang-14.0.6;/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1;/usr/tce/packages/python/python-3.9.12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7/serac-develop-rxm4riztuqljjlnrhyomn4cqusqy5g6z/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: clang@=19.1.7 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/spack/lib/spack/env/clang/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/spack/lib/spack/env/clang/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/spack/lib/spack/env/clang/gfortran" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran" CACHE PATH "") + +endif() + +set(CMAKE_C_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_CXX_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_Fortran_FLAGS "-fPIE -fPIC" CACHE STRING "") + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_06_03_16_52_06/clang-19.1.7" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua" CACHE PATH "") + +set(ARPACK_DIR "${TPL_ROOT}/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-5o3zfbeosigjwrp5ffizjs6sf7pyoqjv" CACHE PATH "") + +set(PETSC_DIR "${TPL_ROOT}/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43" CACHE PATH "") + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv" CACHE PATH "") + +set(SLEPC_DIR "${TPL_ROOT}/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6" CACHE PATH "") + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +set(DEVTOOLS_ROOT "/usr/WS2/smithdev/devtools/toss_4_x86_64_ib/2024_05_30_15_02_20/._view/psk2dcrijss6s4i6qmxplzthrzm3y7nh" CACHE PATH "") + +set(ATS_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/ats" CACHE PATH "") + +set(CLANGFORMAT_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-format" CACHE PATH "") + +set(CLANGTIDY_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-tidy" CACHE PATH "") + +set(ENABLE_DOCS ON CACHE BOOL "") + +set(SPHINX_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/sphinx-build" CACHE PATH "") + +set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH "") + +set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.9.8/bin/doxygen" CACHE PATH "") + + diff --git a/host-configs/dfem/rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake b/host-configs/dfem/rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake new file mode 100644 index 000000000..f7132a3f6 --- /dev/null +++ b/host-configs/dfem/rzvernal-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake @@ -0,0 +1,149 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/libunwind-1.8.1-667lulihqw77lixyhoyezlqadqmw3ufq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/fmt-11.0.2-haismsyfch472jneh7tj5hoz5faug2wp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/netlib-scalapack-2.2.2-zinikcerv4cwfrzt4kbhby6xm76fjmvq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh;/usr/tce;/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1/llvm;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: rocmcc@=6.2.1 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdflang" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdflang" CACHE PATH "") + +endif() + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/global/tools/flux_wrappers/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +#------------------------------------------------ +# ROCm +#------------------------------------------------ + +set(HIP_ROOT_DIR "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_HIP_COMPILER "/opt/rocm-6.2.1/llvm/bin/clang++" CACHE FILEPATH "") + +set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "") + +set(AMDGPU_TARGETS "gfx90a" CACHE STRING "") + +set(GPU_TARGETS "gfx90a" CACHE STRING "") + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +set(ENABLE_HIP ON CACHE BOOL "") + +set(ROCM_PATH "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_EXE_LINKER_FLAGS "-L/opt/rocm-6.2.1/lib -Wl,-rpath,/opt/rocm-6.2.1/lib -L/opt/rocm-6.2.1/llvm/lib -Wl,-rpath,/opt/rocm-6.2.1/llvm/lib -lxpmem -L/opt/cray/pe/mpich/8.1.29/gtl/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.29/gtl/lib -lmpi_gtl_hsa -Wl,--disable-new-dtags -lflang -lflangrti -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr -lpgmath -lhipblas" CACHE STRING "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem" CACHE PATH "") + +# PETSC not built + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp" CACHE PATH "") + +# SLEPC not built + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd" CACHE PATH "") + +set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib_cray/enzyme/rocm-6.2.1/0.0.180" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +# Code checks disabled due to disabled devtools + +set(SERAC_ENABLE_CODE_CHECKS OFF CACHE BOOL "") + +set(ENABLE_CLANGFORMAT OFF CACHE BOOL "") + +set(ENABLE_CLANGTIDY OFF CACHE BOOL "") + +set(ENABLE_DOCS OFF CACHE BOOL "") + +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + diff --git a/host-configs/dfem/rzwhippet-toss_4_x86_64_ib-clang@19.1.7.cmake b/host-configs/dfem/rzwhippet-toss_4_x86_64_ib-clang@19.1.7.cmake new file mode 100644 index 000000000..747b9ce1a --- /dev/null +++ b/host-configs/dfem/rzwhippet-toss_4_x86_64_ib-clang@19.1.7.cmake @@ -0,0 +1,143 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/caliper-2.12.1-a4chnmcpoktn7vlyrjxbw2qskkkytkj3;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/libunwind-1.8.1-tgmphz65mn62bautt2ksl45ckecw5e2c;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/fmt-11.0.2-xpfumigjok66664dhruf6tfaa7gwcn3n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/netlib-scalapack-2.2.2-fuk7tt7il4e6tekxxctc625g4xk4hqeu;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/tce;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/cppcheck-2.9;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/doxygen-1.9.8;/usr/tce/packages/clang/clang-14.0.6;/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1;/usr/tce/packages/python/python-3.9.12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7/serac-develop-t2mywg4ueanbqllbapg5wvrivg3umjoj/lib64;;/usr/tce/packages/gcc/gcc-10.3.1/lib/gcc/x86_64-redhat-linux/10" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: clang@=19.1.7 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/spack/lib/spack/env/clang/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/spack/lib/spack/env/clang/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/spack/lib/spack/env/clang/gfortran" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/toss_4_x86_64_ib/llvm/19.1.7/bin/clang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/tce/packages/gcc/gcc-10.3.1/bin/gfortran" CACHE PATH "") + +endif() + +set(CMAKE_C_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_CXX_FLAGS "--gcc-toolchain=/usr/tce/packages/gcc/gcc-10.3.1" CACHE STRING "") + +set(CMAKE_Fortran_FLAGS "-fPIE -fPIC" CACHE STRING "") + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-10.3.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib/2025_05_27_15_56_53/clang-19.1.7" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-tkmjd26d72eqk4tts7hwlwognvwuyj5s" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-afpyaxeo4ulr2lkljpy2zr7rjeq64cox" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-t2zobt76hkwap7d2ppddp3traorxpffl" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-gbv5tbeosslcqh2zhkcrgkwtqxob257j" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-pu3kif4ltqdw2kcwvlswfze6pmmsjs6q" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-ta6n6j6ziyuw2tfejt27nkdulvjq27hu" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-ebbueyypx5bn5mekybxavligkykhm5fy" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-morhecieft72ri6gi2r6hsatzrgdhe5n" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-62ri63jaeyajwrrvwk75dbwvrnj6fony" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-k53l7hvjwbqnepeb7etlujfgzrzfliua" CACHE PATH "") + +set(ARPACK_DIR "${TPL_ROOT}/arpack-ng-3.9.0-ahgmmge3jagsx3n72wbxtcznbubcclca" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-5wjrv4xwbm3y7yh7745ijc45sbcwqwqi" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-a4chnmcpoktn7vlyrjxbw2qskkkytkj3" CACHE PATH "") + +set(PETSC_DIR "${TPL_ROOT}/petsc-3.21.6-elp7iw6xe64daspgy5ctubfo3jj4nd43" CACHE PATH "") + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-t5ay2skmkw3ybywrq33tcyl5xbn5g3gv" CACHE PATH "") + +set(SLEPC_DIR "${TPL_ROOT}/slepc-3.21.2-rji2oyfq6x4lyovnrkfqnleoc3yrqrm6" CACHE PATH "") + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-zmfwdjw54im5o6m3gqssnkvuagdzljwe" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-wwas67vc5xfgayl2uimersm3xszlqv62" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-47yl3s6ob3ru4g6qnma4tiv6ob5fr6p6" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-my4bcnwnewxcnewg3xgbzvym64dfizuj" CACHE PATH "") + +set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib/enzyme/llvm_19.1.7/0.0.180" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +set(DEVTOOLS_ROOT "/usr/WS2/smithdev/devtools/toss_4_x86_64_ib/2024_05_30_15_09_55/._view/rkqkran3ydsuprr2wip5pdnz5wh7xwnr" CACHE PATH "") + +set(ATS_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/ats" CACHE PATH "") + +set(CLANGFORMAT_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-format" CACHE PATH "") + +set(CLANGTIDY_EXECUTABLE "/usr/tce/packages/clang/clang-14.0.6/bin/clang-tidy" CACHE PATH "") + +set(ENABLE_DOCS ON CACHE BOOL "") + +set(SPHINX_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/sphinx-build" CACHE PATH "") + +set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH "") + +set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.9.8/bin/doxygen" CACHE PATH "") + +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + diff --git a/host-configs/dfem/tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake b/host-configs/dfem/tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake new file mode 100644 index 000000000..4a75768ea --- /dev/null +++ b/host-configs/dfem/tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip_dfem.cmake @@ -0,0 +1,149 @@ +#------------------------------------------------------------------------------ +# !!!! This is a generated file, edit at own risk !!!! +#------------------------------------------------------------------------------ +# CMake executable path: /usr/tce/bin/cmake +#------------------------------------------------------------------------------ + +set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/libunwind-1.8.1-667lulihqw77lixyhoyezlqadqmw3ufq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/fmt-11.0.2-haismsyfch472jneh7tj5hoz5faug2wp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/netlib-scalapack-2.2.2-zinikcerv4cwfrzt4kbhby6xm76fjmvq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh;/usr/tce;/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1/llvm;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "") + +set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "") + +set(CMAKE_BUILD_TYPE "Release" CACHE STRING "") + +#------------------------------------------------------------------------------ +# Compilers +#------------------------------------------------------------------------------ +# Compiler Spec: rocmcc@=6.2.1 +#------------------------------------------------------------------------------ +if(DEFINED ENV{SPACK_CC}) + + set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/spack/lib/spack/env/rocmcc/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/spack/lib/spack/env/rocmcc/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/spack/lib/spack/env/rocmcc/amdflang" CACHE PATH "") + +else() + + set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang" CACHE PATH "") + + set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang++" CACHE PATH "") + + set(CMAKE_Fortran_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdflang" CACHE PATH "") + +endif() + +#------------------------------------------------------------------------------ +# MPI +#------------------------------------------------------------------------------ + +set(MPI_C_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicc" CACHE PATH "") + +set(MPI_CXX_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicxx" CACHE PATH "") + +set(MPI_Fortran_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpif90" CACHE PATH "") + +set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "") + +set(ENABLE_MPI ON CACHE BOOL "") + +set(MPIEXEC_EXECUTABLE "/usr/global/tools/flux_wrappers/bin/srun" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Hardware +#------------------------------------------------------------------------------ + +#------------------------------------------------ +# ROCm +#------------------------------------------------ + +set(HIP_ROOT_DIR "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_HIP_COMPILER "/opt/rocm-6.2.1/llvm/bin/clang++" CACHE FILEPATH "") + +set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "") + +set(AMDGPU_TARGETS "gfx90a" CACHE STRING "") + +set(GPU_TARGETS "gfx90a" CACHE STRING "") + +set(ENABLE_OPENMP OFF CACHE BOOL "") + +set(ENABLE_HIP ON CACHE BOOL "") + +set(ROCM_PATH "/opt/rocm-6.2.1" CACHE PATH "") + +set(CMAKE_EXE_LINKER_FLAGS "-L/opt/rocm-6.2.1/lib -Wl,-rpath,/opt/rocm-6.2.1/lib -L/opt/rocm-6.2.1/llvm/lib -Wl,-rpath,/opt/rocm-6.2.1/llvm/lib -lxpmem -L/opt/cray/pe/mpich/8.1.29/gtl/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.29/gtl/lib -lmpi_gtl_hsa -Wl,--disable-new-dtags -lflang -lflangrti -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr -lpgmath -lhipblas" CACHE STRING "") + +#------------------------------------------------------------------------------ +# TPLs +#------------------------------------------------------------------------------ + +set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_36/rocmcc-6.2.1" CACHE PATH "") + +set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg" CACHE PATH "") + +set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq" CACHE PATH "") + +set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj" CACHE PATH "") + +set(LUA_DIR "/usr" CACHE PATH "") + +set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd" CACHE PATH "") + +set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6" CACHE PATH "") + +set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh" CACHE PATH "") + +set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh" CACHE PATH "") + +set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev" CACHE PATH "") + +set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt" CACHE PATH "") + +set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5" CACHE PATH "") + +set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh" CACHE PATH "") + +# AMGX not built + +set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem" CACHE PATH "") + +# PETSC not built + +set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp" CACHE PATH "") + +# SLEPC not built + +set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg" CACHE PATH "") + +set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr" CACHE PATH "") + +set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37" CACHE PATH "") + +set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd" CACHE PATH "") + +set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib_cray/enzyme/rocm-6.2.1/0.0.180" CACHE PATH "") + +#------------------------------------------------------------------------------ +# Devtools +#------------------------------------------------------------------------------ + +# Code checks disabled due to disabled devtools + +set(SERAC_ENABLE_CODE_CHECKS OFF CACHE BOOL "") + +set(ENABLE_CLANGFORMAT OFF CACHE BOOL "") + +set(ENABLE_CLANGTIDY OFF CACHE BOOL "") + +set(ENABLE_DOCS OFF CACHE BOOL "") + +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + diff --git a/host-configs/rzwhippet-toss_4_x86_64_ib-llvm@19.1.3.cmake b/host-configs/rzwhippet-toss_4_x86_64_ib-llvm@19.1.3.cmake index 28f32d1b4..2217330f0 100644 --- a/host-configs/rzwhippet-toss_4_x86_64_ib-llvm@19.1.3.cmake +++ b/host-configs/rzwhippet-toss_4_x86_64_ib-llvm@19.1.3.cmake @@ -131,4 +131,9 @@ set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.9.8/bin/doxygen" CACHE PATH "") +set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "") + +set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "") + +set(SERAC_USE_ENZYME ON CACHE BOOL "") diff --git a/mfem b/mfem index fb7466c81..c645943fc 160000 --- a/mfem +++ b/mfem @@ -1 +1 @@ -Subproject commit fb7466c81925eb9a7b5614fa54d5a8bd9395ee36 +Subproject commit c645943fc4ce3020247fbd4c6ce3847cdee188cf diff --git a/scripts/uberenv b/scripts/uberenv index 521848167..e1b395839 160000 --- a/scripts/uberenv +++ b/scripts/uberenv @@ -1 +1 @@ -Subproject commit 521848167c6a1e25aebf55e40d0ec51d7027108c +Subproject commit e1b3958395f9d3df7c33fcb7748c8c1af108fa73 diff --git a/src/smith/infrastructure/accelerator.cpp b/src/smith/infrastructure/accelerator.cpp index 3415307eb..776cd7ce2 100644 --- a/src/smith/infrastructure/accelerator.cpp +++ b/src/smith/infrastructure/accelerator.cpp @@ -21,13 +21,23 @@ namespace { std::unique_ptr device; } // namespace -void initializeDevice() +void initializeDevice(ExecutionSpace exec_space) { SLIC_ERROR_ROOT_IF(device, "smith::accelerator::initializeDevice cannot be called more than once"); device = std::make_unique(); + switch (exec_space) { + case ExecutionSpace::GPU: #if defined(MFEM_USE_CUDA) && defined(SMITH_USE_CUDA_KERNEL_EVALUATION) - device->Configure("cuda"); + device->Configure("cuda"); +#elif defined(MFEM_USE_HIP) + device->Configure("hip"); #endif + break; + case ExecutionSpace::CPU: + break; + case ExecutionSpace::Dynamic: + break; + } } void terminateDevice() diff --git a/src/smith/infrastructure/accelerator.hpp b/src/smith/infrastructure/accelerator.hpp index 034e02697..76400de4e 100644 --- a/src/smith/infrastructure/accelerator.hpp +++ b/src/smith/infrastructure/accelerator.hpp @@ -7,43 +7,58 @@ /** * @file accelerator.hpp * - * @brief This file contains the interface used for initializing/terminating - * any hardware accelerator-related functionality + * @brief This file contains the interface used for initializing/terminating any hardware accelerator-related + * functionality */ #pragma once -#if defined(__CUDACC__) +#include "smith/smith_config.hpp" + +#if defined(SMITH_USE_CUDA) || defined(SMITH_USE_HIP) +/** + * @brief Macro that evaluates to `__host__ __device__` when compiling with nvcc or amdclang and does nothing on a host + * compiler. + */ #define SMITH_HOST_DEVICE __host__ __device__ +/** + * @brief Macro that evaluates to `__host__` when compiling with nvcc or amdclang and does nothing on a host compiler + */ #define SMITH_HOST __host__ +/** + * @brief Macro that evaluates to `__host__` when compiling with nvcc or amdclang and does nothing on a host compiler + */ #define SMITH_DEVICE __device__ +#else +/** + * @brief Macro that evaluates to `__host__ __device__` when compiling with nvcc or amdclang and does nothing on a host + * compiler. + */ +#define SMITH_HOST_DEVICE +/** + * @brief Macro that evaluates to `__host__` when compiling with nvcc or amdclang and does nothing on a host compiler + */ +#define SMITH_HOST +/** + * @brief Macro that evaluates to `__device__` when compiling with nvcc or amdclang and does nothing on a host compiler + */ +#define SMITH_DEVICE +#endif /** * Note: nvcc will sometimes emit a warning if a __host__ __device__ function calls a __host__-only or __device__-only * function. make_tensor is marked __host__ __device__ and is used frequently in the code base, so it was emitting a lot - * of warnings. This #pragma directive suppresses the warning for a specific function. + * of warnings. This pragma directive suppresses the warning for a specific function. */ +#if defined(__CUDACC__) #if __CUDAVER__ >= 75000 #define SMITH_SUPPRESS_NVCC_HOSTDEVICE_WARNING #pragma nv_exec_check_disable #else #define SMITH_SUPPRESS_NVCC_HOSTDEVICE_WARNING #pragma hd_warning_disable #endif - #include -#else //__CUDACC__ -/** - * @brief Macro that evaluates to `__host__ __device__` when compiling with nvcc and does nothing on a host compiler. - */ -#define SMITH_HOST_DEVICE -/** - * @brief Macro that evaluates to `__host__` when compiling with nvcc and does nothing on a host compiler - */ -#define SMITH_HOST -/** - * @brief Macro that evaluates to `__device__` when compiling with nvcc and does nothing on a host compiler - */ -#define SMITH_DEVICE +#else /** * @brief Macro to turn off specific nvcc warnings */ @@ -202,7 +217,7 @@ namespace accelerator { * * @note This function should only be called once */ -void initializeDevice(); +void initializeDevice(ExecutionSpace exec_space); /** * @brief Cleans up the device, if applicable diff --git a/src/smith/infrastructure/application_manager.cpp b/src/smith/infrastructure/application_manager.cpp index 372acb140..1240d9c61 100644 --- a/src/smith/infrastructure/application_manager.cpp +++ b/src/smith/infrastructure/application_manager.cpp @@ -78,7 +78,9 @@ void finalizer() accelerator::terminateDevice(); } -ApplicationManager::ApplicationManager(int argc, char* argv[], MPI_Comm comm, bool doesPrintRunInfo) : comm_(comm) +ApplicationManager::ApplicationManager(int argc, char* argv[], MPI_Comm comm, bool doesPrintRunInfo, + ExecutionSpace exec_space) + : comm_(comm) { // Initialize MPI if (MPI_Init(&argc, &argv) != MPI_SUCCESS) { @@ -120,7 +122,7 @@ ApplicationManager::ApplicationManager(int argc, char* argv[], MPI_Comm comm, bo #endif // Initialize GPU (no-op if not enabled/available) - accelerator::initializeDevice(); + accelerator::initializeDevice(exec_space); // Register signal handlers std::signal(SIGABRT, signalHandler); diff --git a/src/smith/infrastructure/application_manager.hpp b/src/smith/infrastructure/application_manager.hpp index e9885b494..947df5041 100644 --- a/src/smith/infrastructure/application_manager.hpp +++ b/src/smith/infrastructure/application_manager.hpp @@ -11,6 +11,7 @@ #include #include "mpi.h" +#include "smith/infrastructure/accelerator.hpp" namespace smith { @@ -27,8 +28,10 @@ class ApplicationManager { * @param argv The command-line arguments, as C-strings * @param comm The MPI communicator to initialize with * @param doesPrintRunInfo Whether or not to print build information + * @param exec_space The desired execution space of device-capable lambda functions */ - ApplicationManager(int argc, char* argv[], MPI_Comm comm = MPI_COMM_WORLD, bool doesPrintRunInfo = true); + ApplicationManager(int argc, char* argv[], MPI_Comm comm = MPI_COMM_WORLD, bool doesPrintRunInfo = true, + ExecutionSpace exec_space = ExecutionSpace::CPU); /** * @brief Calls smith::finalizer diff --git a/src/smith/numerics/functional/element_restriction.cpp b/src/smith/numerics/functional/element_restriction.cpp index 1b1fee9e8..e31564533 100644 --- a/src/smith/numerics/functional/element_restriction.cpp +++ b/src/smith/numerics/functional/element_restriction.cpp @@ -280,7 +280,7 @@ axom::Array GetElementRestriction(cons } else { uint64_t dofs_per_elem = elem_dofs.size() / n; axom::Array output(n, dofs_per_elem); - std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); + std::memcpy(static_cast(output.data()), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } @@ -346,7 +346,7 @@ axom::Array GetElementDofs(const smith } else { uint64_t dofs_per_elem = elem_dofs.size() / n; axom::Array output(n, dofs_per_elem); - std::memcpy(output.data(), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); + std::memcpy(static_cast(output.data()), elem_dofs.data(), sizeof(DoF) * n * dofs_per_elem); return output; } } @@ -461,7 +461,7 @@ axom::Array GetFaceDofs(const smith::f } else { uint64_t dofs_per_face = face_dofs.size() / n; axom::Array output(n, dofs_per_face); - std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); + std::memcpy(static_cast(output.data()), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } } @@ -634,7 +634,7 @@ axom::Array GetFaceDofs(const smith::f } else { uint64_t dofs_per_face = face_dofs.size() / n; axom::Array output(n, dofs_per_face); - std::memcpy(output.data(), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); + std::memcpy(static_cast(output.data()), face_dofs.data(), sizeof(DoF) * n * dofs_per_face); return output; } } diff --git a/src/smith/physics/CMakeLists.txt b/src/smith/physics/CMakeLists.txt index c85ccd052..19066e61a 100644 --- a/src/smith/physics/CMakeLists.txt +++ b/src/smith/physics/CMakeLists.txt @@ -22,6 +22,9 @@ set(physics_headers base_physics.hpp mesh.hpp common.hpp + dfem_mass_weak_form.hpp + dfem_solid_weak_form.hpp + dfem_weak_form.hpp field_types.hpp fit.hpp heat_transfer.hpp diff --git a/src/smith/physics/contact/contact_data.hpp b/src/smith/physics/contact/contact_data.hpp index c815af3dd..e4fa46757 100644 --- a/src/smith/physics/contact/contact_data.hpp +++ b/src/smith/physics/contact/contact_data.hpp @@ -259,6 +259,7 @@ class ContactData { */ int num_pressure_dofs_; +#ifdef SMITH_USE_TRIBOL /** * @brief Tracks whether the Jacobian and pressure offsets need to be updated * @@ -266,6 +267,7 @@ class ContactData { * */ mutable bool offsets_up_to_date_; +#endif /** * @brief Offsets giving size of each block Jacobian contribution @@ -296,9 +298,11 @@ class ContactData { */ mutable mfem::Array global_pressure_dof_offsets_; +#ifdef SMITH_USE_TRIBOL int cycle_{0}; double time_{0.0}; double dt_{1.0}; +#endif }; } // namespace smith diff --git a/src/smith/physics/dfem_mass_weak_form.hpp b/src/smith/physics/dfem_mass_weak_form.hpp new file mode 100644 index 000000000..b87cb229c --- /dev/null +++ b/src/smith/physics/dfem_mass_weak_form.hpp @@ -0,0 +1,129 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Smith Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/** + * @file dfem_mass_weak_form.hpp + */ + +#pragma once + +#include "smith/smith_config.hpp" + +#ifdef SMITH_USE_DFEM + +#include "mfem.hpp" + +#include "smith/physics/dfem_weak_form.hpp" +#include "smith/physics/mesh.hpp" +#include "smith/physics/state/finite_element_state.hpp" +#include "smith/physics/boundary_conditions/boundary_condition_manager.hpp" + +namespace smith { + +/** + * @brief + * + */ +class LumpedMassExplicitNewmark { + public: + LumpedMassExplicitNewmark(const std::shared_ptr& weak_form, const std::shared_ptr& mass_weak_form, + std::shared_ptr bc_manager) + : weak_form_(weak_form), mass_weak_form_(mass_weak_form), bc_manager_(bc_manager) + { + } + + std::tuple, double> advanceState(const std::vector& states, + const std::vector& params, + double time, double dt) + { + SLIC_ERROR_ROOT_IF(states.size() != 4, "Expected 4 states: displacement, velocity, acceleration, and coordinates"); + + enum States + { + DISPLACEMENT, + VELOCITY, + ACCELERATION, + COORDINATES + }; + + enum Params + { + DENSITY + }; + + const auto& u = *states[DISPLACEMENT]; + const auto& v = *states[VELOCITY]; + const auto& a = *states[ACCELERATION]; + + auto v_pred = v; + v_pred.Add(0.5 * dt, a); + auto u_pred = u; + u_pred.Add(dt, v_pred); + + if (bc_manager_) { + u_pred.SetSubVector(bc_manager_->allEssentialTrueDofs(), 0.0); + } + + auto m_inv = mass_weak_form_->residual(time, dt, &u, {states[COORDINATES], params[DENSITY]}); + m_inv.Reciprocal(); + + std::vector pred_states = {&u_pred, &v_pred, &a, states[COORDINATES], params[DENSITY]}; + + auto zero_mass_resid = weak_form_->residual(time, dt, &u_pred, pred_states); + + FiniteElementState a_pred(a.space(), "acceleration_pred"); + auto a_pred_ptr = a_pred.Write(); + auto m_inv_ptr = m_inv.Read(); + auto zero_mass_resid_ptr = zero_mass_resid.Read(); + mfem::forall_switch(a_pred.UseDevice(), a_pred.Size(), + [=] MFEM_HOST_DEVICE(int i) { a_pred_ptr[i] = m_inv_ptr[i] * zero_mass_resid_ptr[i]; }); + + v_pred.Add(0.5 * dt, a_pred); + + return {{u_pred, v_pred, a_pred, *states[COORDINATES]}, time + dt}; + } + + private: + std::shared_ptr weak_form_; + std::shared_ptr mass_weak_form_; + std::shared_ptr bc_manager_; +}; + +template +auto create_solid_mass_weak_form(const std::string& physics_name, std::shared_ptr& mesh, + const FiniteElementState& lumped_field, const FiniteElementState& density, + const mfem::IntegrationRule& ir) +{ + enum FieldIDs + { + COORDINATES, + DENSITY, + TEST + }; + + auto residual = std::make_shared( + physics_name, mesh, lumped_field.space(), + DfemWeakForm::SpacesT{static_cast(mesh->mfemParMesh().GetNodes()->FESpace()), + &density.space()}); + + mfem::future::tuple, mfem::future::Weight, mfem::future::Value> + mass_integral_inputs{}; + mfem::future::tuple> mass_integral_outputs{}; + + residual->addBodyIntegral( + mesh->mfemParMesh().attributes, + [](mfem::future::tensor dX_dxi, mfem::real_t weight, double rho) { + auto ones = mfem::future::make_tensor([](int) { return 1.0; }); + auto J = mfem::future::det(dX_dxi) * weight; + return mfem::future::tuple{rho * ones * J}; + }, + mass_integral_inputs, mass_integral_outputs, ir, std::index_sequence<>{}); + return residual; +} + +} // namespace smith + +#endif diff --git a/src/smith/physics/dfem_solid_weak_form.hpp b/src/smith/physics/dfem_solid_weak_form.hpp new file mode 100644 index 000000000..f63ccd622 --- /dev/null +++ b/src/smith/physics/dfem_solid_weak_form.hpp @@ -0,0 +1,238 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Smith Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/** + * @file dfem_solid_weak_form.hpp + * + * @brief Implements the WeakForm interface for solid mechanics physics using dFEM. Derives from DfemWeakForm. + */ + +#pragma once + +#include +#include "smith/smith_config.hpp" + +#ifdef SMITH_USE_DFEM + +#include "smith/physics/dfem_weak_form.hpp" + +#include "smith/infrastructure/accelerator.hpp" + +namespace smith { + +template +struct ScalarParameter { + static constexpr int index = Idx; + using QFunctionInput = double; + template + using QFunctionFieldOp = mfem::future::Value; +}; + +template +struct InternalVariableParameter { + static constexpr int index = Idx; + using QFunctionInput = mfem::future::tensor; + template + using QFunctionFieldOp = mfem::future::Identity; +}; + +template +struct StressDivQFunction { + SMITH_HOST_DEVICE inline auto operator()( + // mfem::real_t dt, // TODO: figure out how to pass this in + const mfem::future::tensor& du_dxi, + const mfem::future::tensor& dv_dxi, + const mfem::future::tensor&, + const mfem::future::tensor& dX_dxi, mfem::real_t weight, + Parameters::QFunctionInput... params) const + { + auto dxi_dX = mfem::future::inv(dX_dxi); + auto du_dX = mfem::future::dot(du_dxi, dxi_dX); + auto dv_dX = mfem::future::dot(dv_dxi, dxi_dX); + double dt = 1.0; // TODO: figure out how to pass this in to the qfunction + auto P = material.pkStress(dt, du_dX, dv_dX, params...); + auto JxW = mfem::future::det(dX_dxi) * weight * mfem::future::transpose(dxi_dX); + return mfem::future::tuple{-P * JxW}; + } + + Material material; ///< the material model to use for computing the stress +}; + +template +struct AccelerationQFunction { + SMITH_HOST_DEVICE inline auto operator()( + // mfem::real_t dt, // TODO: figure out how to pass this in + const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor& a, + const mfem::future::tensor& dX_dxi, mfem::real_t weight, + Parameters::QFunctionInput... params) const + { + auto rho = material.density(params...); + auto J = mfem::future::det(dX_dxi) * weight; + return mfem::future::tuple{-rho * a * J}; + } + + Material material; ///< the material model to use for computing the density +}; + +/** + * @brief The weak form for solid mechanics + * + * This uses dFEM to compute the solid mechanics residuals and tangent stiffness matrices. + */ +template +class DfemSolidWeakForm : public DfemWeakForm { + public: + /// @brief enumeration of the required states + enum STATE + { + DISPLACEMENT, + VELOCITY, + ACCELERATION, + COORDINATES, + NUM_STATES + }; + + /** + * @brief Construct a new DfemSolidWeakForm object + * + * @param physics_name A name for the physics module instance + * @param mesh The smith Mesh + * @param test_space Test space + * @param parameter_fe_spaces Vector of parameter finite element spaces + * @param parameter_quadrature_spaces Vector of parameter quadrature spaces + */ + DfemSolidWeakForm(std::string physics_name, std::shared_ptr mesh, const mfem::ParFiniteElementSpace& test_space, + const std::vector parameter_quadrature_spaces = {}, + const std::vector& parameter_fe_spaces = {}) + : DfemWeakForm(physics_name, mesh, test_space, makeInputSpaces(test_space, mesh, parameter_fe_spaces), + parameter_quadrature_spaces) + { + } + + /** + * @brief Set the material stress response and mass properties for the physics module + * + * @tparam MaterialType The solid material type + * @tparam ParameterTypes types that contains the internal variables for MaterialType + * @param domain_attributes Array of MFEM element attributes over which to compute the integral + * @param material A material that provides a function to evaluate PK1 stress + * @pre material must be a object that has a pkStress method with the following arguments: + * 1. `double dt` the timestep size + * 2. `tensor du_dX` the displacement gradient at this quadrature point + * 3. `tensor dv_dX` the velocity gradient at this quadrature point + * 4. Additional arguments for the dependent parameters of the material + * + * @pre MaterialType must have a public method `density` which can take FiniteElementState parameter inputs + * @pre MaterialType must have a public method 'pkStress' which returns the first Piola-Kirchhoff stress + * + */ + template + void setMaterial(const mfem::Array& domain_attributes, const MaterialType& material, + const mfem::IntegrationRule& displacement_ir) + { + SLIC_ERROR_IF(material.dim != DfemWeakForm::mesh_->mfemParMesh().Dimension(), + "Material model dimension does not match mesh dimension."); + + auto stress_div_integral = StressDivQFunction{.material = material}; + mfem::future::tuple, mfem::future::Gradient, + mfem::future::Gradient, mfem::future::Gradient, mfem::future::Weight, + typename ParameterTypes::template QFunctionFieldOp...> + stress_div_integral_inputs{}; + + // BT Why is this not Gradient ? + mfem::future::tuple> stress_div_integral_outputs{}; + + DfemWeakForm::addBodyIntegral(domain_attributes, stress_div_integral, stress_div_integral_inputs, + stress_div_integral_outputs, displacement_ir, + std::index_sequence{}); + + if constexpr (!IsQuasiStatic) { + auto acceleration_integral = AccelerationQFunction{.material = material}; + mfem::future::tuple, mfem::future::Value, + mfem::future::Value, mfem::future::Gradient, mfem::future::Weight, + typename ParameterTypes::template QFunctionFieldOp...> + acceleration_integral_inputs{}; + mfem::future::tuple> acceleration_integral_outputs{}; + if constexpr (UseLumpedMass) { + SLIC_ERROR_IF(DfemWeakForm::input_mfem_spaces_[DISPLACEMENT]->IsVariableOrder(), + "Lumped mass matrix is not supported for variable order finite element spaces."); + auto& mesh = DfemWeakForm::mesh_->mfemParMesh(); + SLIC_ERROR_IF(mesh.GetNumGeometries(mesh.Dimension()) != 1 || + !(mesh.HasGeometry(mfem::Geometry::SQUARE) || mesh.HasGeometry(mfem::Geometry::CUBE)), + "Lumped mass matrix is only supported for 2D and 3D meshes with square or cubic elements."); + // use lumped mass matrix via integration rule at nodes + auto& fe_coll = *DfemWeakForm::input_mfem_spaces_[DISPLACEMENT]->FEColl(); + mfem::IntegrationRule rule_1d; + mfem::QuadratureFunctions1D::GaussLobatto(fe_coll.GetOrder() + 1, &rule_1d); + auto spatial_dim = DfemWeakForm::input_mfem_spaces_[DISPLACEMENT]->GetVDim(); + switch (spatial_dim) { + case 1: + nodal_ir_ = std::make_unique(rule_1d); + break; + case 2: + nodal_ir_ = std::make_unique(rule_1d, rule_1d); + break; + case 3: + nodal_ir_ = std::make_unique(rule_1d, rule_1d, rule_1d); + break; + default: + SLIC_ERROR_ROOT("Unsupported number of dimensions for nodal integration rule."); + } + DfemWeakForm::addBodyIntegral(domain_attributes, acceleration_integral, acceleration_integral_inputs, + acceleration_integral_outputs, *nodal_ir_, std::index_sequence{}); + } else { + // use consistent mass matrix + DfemWeakForm::addBodyIntegral(domain_attributes, acceleration_integral, acceleration_integral_inputs, + acceleration_integral_outputs, displacement_ir, + std::index_sequence{}); + } + } + } + + void massMatrix(const std::vector& fields, const mfem::Vector& direction_t, + mfem::Vector& result_t) const + { + static_assert(!IsQuasiStatic, "Mass matrix is not defined for quasi-static solid mechanics problems."); + // Pass empty quad field vector, assuming mass matrix cannot depend on internal state variables + auto deriv_op = DfemWeakForm::weak_form_.GetDerivative(ACCELERATION, {&fields[0]->gridFunction()}, + DfemWeakForm::getLVectors(fields, {})); + deriv_op->Mult(direction_t, result_t); + } + + protected: + std::unique_ptr nodal_ir_; + + private: + /** + * @brief Creates a list of MFEM input spaces compatible with dFEM + * + * @param test_space Space the q-function will be integrated against + * @param mesh Problem mesh + * @param parameter_fe_spaces Vector of finite element spaces which are parameter arguments to the residual + * @return Vector of input spaces that are passed along to the dFEM differentiable operator constructor + */ + std::vector makeInputSpaces( + const mfem::ParFiniteElementSpace& test_space, const std::shared_ptr& mesh, + const std::vector& parameter_fe_spaces) + { + std::vector input_spaces; + input_spaces.reserve(4 + parameter_fe_spaces.size()); + for (int i = 0; i < 3; ++i) { + input_spaces.push_back(&test_space); + } + input_spaces.push_back(static_cast(mesh->mfemParMesh().GetNodalFESpace())); + for (auto space : parameter_fe_spaces) { + input_spaces.push_back(space); + } + return input_spaces; + } +}; + +} // namespace smith + +#endif diff --git a/src/smith/physics/dfem_weak_form.hpp b/src/smith/physics/dfem_weak_form.hpp new file mode 100644 index 000000000..fd358f314 --- /dev/null +++ b/src/smith/physics/dfem_weak_form.hpp @@ -0,0 +1,349 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Smith Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +/** + * @file dfem_weak_form.hpp + * + * @brief Implements the WeakForm interface using dfem. Allows for generic specification of body and boundary integrals + */ + +#pragma once + +#include "smith/smith_config.hpp" + +#ifdef SMITH_USE_DFEM + +#include "smith/physics/weak_form.hpp" +#include "smith/physics/mesh.hpp" +#include "smith/physics/state/finite_element_state.hpp" + +// NOTE (EBC): these should be upstreamed to MFEM, so let's put them in the mfem::future namespace +namespace mfem { +namespace future { + +// Inner product of 1D tensors +template +MFEM_HOST_DEVICE auto inner(const tensor& A, const tensor& B) -> decltype(S{} * T{}) +{ + decltype(S{} * T{}) sum{}; + for (int i = 0; i < m; i++) { + sum += A[i] * B[i]; + } + return sum; +} + +/** + * @brief computes det(A + I) - 1, where precision is not lost when the entries A_{ij} << 1 + * + * detApIm1(A) = det(A + I) - 1 + * When the entries of A are small compared to unity, computing + * det(A + I) - 1 directly will suffer from catastrophic cancellation. + * + * @param A Input matrix + * @return det(A + I) - 1, where I is the identity matrix + */ +template +MFEM_HOST_DEVICE constexpr auto detApIm1(const mfem::future::tensor& A) +{ + // From the Cayley-Hamilton theorem, we get that for any N by N matrix A, + // det(A - I) - 1 = I1(A) + I2(A) + ... + IN(A), + // where the In are the principal invariants of A. + // We inline the definitions of the principal invariants to increase computational speed. + + // equivalent to tr(A) + det(A) + return A(0, 0) - A(0, 1) * A(1, 0) + A(1, 1) + A(0, 0) * A(1, 1); +} + +/// @overload +template +MFEM_HOST_DEVICE constexpr auto detApIm1(const mfem::future::tensor& A) +{ + // For notes on the implementation, see the 2x2 version. + + // clang-format off + // equivalent to tr(A) + I2(A) + det(A) + return A(0, 0) + A(1, 1) + A(2, 2) + - A(0, 1) * A(1, 0) * (1 + A(2, 2)) + + A(0, 0) * A(1, 1) * (1 + A(2, 2)) + - A(0, 2) * A(2, 0) * (1 + A(1, 1)) + - A(1, 2) * A(2, 1) * (1 + A(0, 0)) + + A(0, 0) * A(2, 2) + + A(1, 1) * A(2, 2) + + A(0, 1) * A(1, 2) * A(2, 0) + + A(0, 2) * A(1, 0) * A(2, 1); + // clang-format on +} + +} // namespace future +} // namespace mfem + +namespace smith { + +// NOTE: Args needs to be on the functor struct instead of the operator() so that operator() isn't overloaded and dfem +// can deduce the type +/** + * @brief Helper struct to build a (typically scalar) q-function as the inner product of an existing q-function output + * with a given tensor of the same type + */ +template +struct InnerQFunction { + InnerQFunction(OrigQFn orig_qfn) : orig_qfn_(orig_qfn) {} + + SMITH_HOST_DEVICE inline auto operator()(Primal V, Args... args) const + { + Primal orig_residual = mfem::future::get<0>(orig_qfn_(std::forward(args)...)); + return mfem::future::tuple{mfem::future::inner(V, orig_residual)}; + } + + OrigQFn orig_qfn_; +}; + +// Step 2: deduce the type of the parameters and the first tuple element of the return type of the operator() +// Step 3: create the InnerQFunction with the deduced types +template +auto makeInnerQFunction(OrigQFn orig_qfn, R (OrigQFn::*)(Args...) const) +{ + // TODO: is there a better way to get the type of the first tuple element? + return InnerQFunction(R{})), OrigQFn, Args...>{orig_qfn}; +} + +// Step 1: get function pointer to operator() +template +auto makeInnerQFunction(OrigQFn orig_qfn) +{ + return makeInnerQFunction(orig_qfn, &OrigQFn::operator()); +} + +/** + * @brief A nonlinear WeakForm class implemented using dfem + * + * This uses dfem to compute fairly general residuals and tangent stiffness matrices based on body and boundary weak + * form integrals. + * + */ +class DfemWeakForm : public WeakForm { + public: + using SpacesT = std::vector; ///< typedef + + /** + * @brief Construct a new DfemWeakForm object + * + * @param physics_name A name for the physics module instance + * @param mesh The smith mesh + * @param output_mfem_space Test space + * @param input_mfem_spaces Vector of finite element spaces which are arguments to the residual + */ + DfemWeakForm(std::string physics_name, std::shared_ptr mesh, const mfem::ParFiniteElementSpace& output_fe_space, + const SpacesT& input_fe_spaces, + const std::vector& input_quadrature_spaces = {}) + : WeakForm(physics_name), + mesh_(mesh), + output_mfem_space_(output_fe_space), + input_mfem_spaces_(input_fe_spaces), + weak_form_( + makeFieldDescriptors({&output_fe_space}, {}, input_fe_spaces.size() + input_quadrature_spaces.size()), + makeFieldDescriptors(input_fe_spaces, input_quadrature_spaces), mesh->mfemParMesh()), + v_dot_weak_form_residual_( + makeFieldDescriptors({&output_fe_space}, {}, input_fe_spaces.size() + input_quadrature_spaces.size()), + makeFieldDescriptors(input_fe_spaces, input_quadrature_spaces), mesh->mfemParMesh()), + residual_vector_(output_fe_space.GetTrueVSize()) + { + // sum field operator doesn't work with sum factorization + v_dot_weak_form_residual_.DisableTensorProductStructure(); + residual_vector_.UseDevice(true); + } + + /** + * @brief Add a body integral contribution to the residual + * + * @tparam BodyIntegralType The type of the body integral + * @tparam InputType mfem::future::tuple holding mfem::future::FieldOperator of the body integral inputs + * @tparam OutputType mfem::future::tuple holding the single mfem::future::FieldOperator of the body integral output + * @tparam DerivIdsType std::index_sequence of field IDs where derivatives are needed + * @param domain_attributes Array of MFEM element attributes over which to compute the integral + * @param body_integral A function describing the body force applied. Our convention for the sign of the residual + * vector is that it is expected to be a 'negative force', so the mass terms show up with a positive sign in the + * residual. This also ensures that the Jacobian of the residual is positive definite for most physics. A body + * integrand involving 'right hand side' contributions like a body load, should be supplied by the user with a + * negative sign. + * @param integral_inputs Empty InputType + * @param integral_outputs Empty OutputType + * @param integration_rule Integration rule to use on each element + * @param derivative_ids Empty DerivIdsType + * + * @pre body_integral must take all fields as either an InputType or an OutputType + * + */ + template + void addBodyIntegral(mfem::Array domain_attributes, BodyIntegralType body_integral, InputType integral_inputs, + OutputType integral_outputs, const mfem::IntegrationRule& integration_rule, + DerivIdsType derivative_ids) + { + weak_form_.AddDomainIntegrator(body_integral, integral_inputs, integral_outputs, integration_rule, + domain_attributes, derivative_ids); + auto scalar_body_integral = makeInnerQFunction(body_integral); + v_dot_weak_form_residual_.AddDomainIntegrator( + scalar_body_integral, addToTupleType(integral_inputs, mfem::future::get<0>(integral_outputs)), + mfem::future::tuple(integral_outputs).GetFieldId()>>{}, integration_rule, + domain_attributes, derivative_ids); + } + + /// @overload + mfem::Vector residual(double /*time*/, double dt, ConstFieldPtr /*shape_disp*/, + const std::vector& fields, + const std::vector& quad_fields = {}, int block_row = 0) const override + { + SLIC_ERROR_ROOT_IF(block_row != 0, "Invalid block row and column requested in fieldJacobian for DfemWeakForm"); + + dt_ = dt; + + weak_form_.SetParameters(getLVectors(fields, quad_fields)); + weak_form_.Mult(residual_vector_, residual_vector_); + return residual_vector_; + } + + /// @overload + std::unique_ptr jacobian(double /*time*/, double dt, ConstFieldPtr /*shape_disp*/, + const std::vector& /*fields*/, + const std::vector& /*jacobian_weights*/, + const std::vector& /*quad_fields*/ = {}, + int /*block_row*/ = 0) const override + { + SLIC_ERROR_ROOT("DfemWeakForm does not support matrix assembly"); + + dt_ = dt; + + return std::make_unique(); + } + + /// @overload + void jvp(double /*time*/, double dt, ConstFieldPtr /*shape_disp*/, const std::vector& fields, + const std::vector& quad_fields, ConstFieldPtr /*v_shape_disp*/, + const std::vector& v_fields, const std::vector& /*v_quad_fields*/, + const std::vector& jvp_reactions) const override + { + SLIC_ERROR_IF(v_fields.size() != fields.size(), + "Invalid number of field sensitivities relative to the number of fields"); + SLIC_ERROR_IF(jvp_reactions.size() != 1, "FunctionalResidual nonlinear systems only supports 1 output residual"); + + dt_ = dt; + + std::vector test_par_gf({&fields[0]->gridFunction()}); + std::vector field_par_gf = getLVectors(fields, quad_fields); + + *jvp_reactions[0] = 0.0; + + for (size_t input_col = 0; input_col < fields.size(); ++input_col) { + if (v_fields[input_col] != nullptr) { + auto deriv_op = weak_form_.GetDerivative(input_col, test_par_gf, field_par_gf); + deriv_op->AddMult(*v_fields[input_col], *jvp_reactions[0]); + } + } + } + + /// @overload + void vjp(double /*time*/, double dt, ConstFieldPtr /*shape_disp*/, const std::vector& fields, + const std::vector& quad_fields, const std::vector& v_fields, + DualFieldPtr /*vjp_shape_disp_sensitivity*/, const std::vector& vjp_sensitivities, + const std::vector& /*vjp_quad_field_sensitivities*/) const override + { + SLIC_ERROR_IF(vjp_sensitivities.size() != fields.size(), + "Invalid number of field sensitivities relative to the number of fields"); + SLIC_ERROR_IF(v_fields.size() != 1, "FunctionalResidual nonlinear systems only supports 1 output residual"); + + dt_ = dt; + + std::vector test_par_gf({&v_fields[0]->gridFunction()}); + std::vector field_par_gf = getLVectors(fields, quad_fields); + // field_par_gf.push_back(&v_fields[0]->gridFunction()); + + for (size_t input_col = 0; input_col < fields.size(); ++input_col) { + if (vjp_sensitivities[input_col] != nullptr) { + auto deriv_op = v_dot_weak_form_residual_.GetDerivative(input_col, test_par_gf, field_par_gf); + // do this entry by entry until assembly is supported + mfem::Vector direction(vjp_sensitivities[input_col]->Size()); + direction = 0.0; + for (int i = 0; i < vjp_sensitivities[input_col]->Size(); ++i) { + direction[i] = 1.0; + mfem::Vector value(1); + deriv_op->Mult(direction, value); + (*vjp_sensitivities[input_col])[i] += value[0]; + direction[i] = 0.0; + } + } + } + } + + protected: + static std::vector makeFieldDescriptors( + const std::vector& fe_spaces, + const std::vector& quad_spaces, size_t offset = 0) + { + std::vector field_descriptors; + field_descriptors.reserve(fe_spaces.size() + quad_spaces.size()); + for (size_t i = 0; i < fe_spaces.size(); ++i) { + field_descriptors.emplace_back(i + offset, fe_spaces[i]); + } + for (size_t i = 0; i < quad_spaces.size(); ++i) { + field_descriptors.emplace_back(i + offset + fe_spaces.size(), quad_spaces[i]); + } + return field_descriptors; + } + + std::vector getLVectors(const std::vector& fields, + const std::vector& quad_fields) const + { + std::vector fields_l; + fields_l.reserve(fields.size() + quad_fields.size()); + for (size_t i = 0; i < fields.size(); ++i) { + fields_l.push_back(&fields[i]->gridFunction()); + } + for (size_t i = 0; i < quad_fields.size(); ++i) { + // dfem interface wants non-const vectors (but it should not modify them) + // TODO (EBC): fix this in dfem interface + fields_l.push_back(const_cast(quad_fields[i])); + } + return fields_l; + } + + template + static auto addToTupleType(const mfem::future::tuple&, const Tnew&) + { + return mfem::future::tuple{}; + } + + // The field ID doesn't matter, since the test function is one + template class FieldOp> + static auto makeVirtualWorkOutputs(mfem::future::tuple>) + { + return mfem::future::tuple>{}; + } + + /// @brief timestep, this needs to be held here and modified for rate dependent applications + mutable double dt_ = std::numeric_limits::max(); + + /// @brief primary mesh + std::shared_ptr mesh_; + + /// @brief Output field (test) space + const mfem::ParFiniteElementSpace& output_mfem_space_; + + /// @brief Input field (trial) spaces + std::vector input_mfem_spaces_; + + /// @brief dfem residual evaluator + mutable mfem::future::DifferentiableOperator weak_form_; + + /// @brief dfem residual times an arbitrary vector v (same space as residual) evaluator + mutable mfem::future::DifferentiableOperator v_dot_weak_form_residual_; + + /// @brief residual vector + mutable mfem::Vector residual_vector_; +}; + +} // namespace smith + +#endif diff --git a/src/smith/physics/state/finite_element_vector.cpp b/src/smith/physics/state/finite_element_vector.cpp index a3778948e..e7f4e9ca5 100644 --- a/src/smith/physics/state/finite_element_vector.cpp +++ b/src/smith/physics/state/finite_element_vector.cpp @@ -50,6 +50,9 @@ FiniteElementVector::FiniteElementVector(const mfem::ParFiniteElementSpace& spac // Initialize the vector to zero HypreParVector::operator=(0.0); + + // Use device (if available) + UseDevice(true); } FiniteElementVector::FiniteElementVector(FiniteElementVector&& input_vector) diff --git a/src/smith/physics/state/state_manager.cpp b/src/smith/physics/state/state_manager.cpp index a2d8e7447..1749a5669 100644 --- a/src/smith/physics/state/state_manager.cpp +++ b/src/smith/physics/state/state_manager.cpp @@ -209,6 +209,15 @@ void StateManager::save(const double t, const int cycle, const std::string& mesh SMITH_MARK_FUNCTION; SLIC_ERROR_ROOT_IF(!ds_, "Smith's data store was not initialized - call StateManager::initialize first"); SLIC_ERROR_ROOT_IF(!hasMesh(mesh_tag), axom::fmt::format("Mesh tag '{}' not found in the data store", mesh_tag)); + + // copy data to host (if needed; HostRead() does nothing if host data is up-to-date) + for (auto& state : named_states_) { + state.second->HostRead(); + } + for (auto& dual : named_duals_) { + dual.second->HostRead(); + } + auto& datacoll = datacolls_.at(mesh_tag); std::string file_path = axom::utilities::filesystem::joinPath(datacoll.GetPrefixPath(), datacoll.GetCollectionName()); SLIC_INFO_ROOT( diff --git a/src/smith/physics/tests/CMakeLists.txt b/src/smith/physics/tests/CMakeLists.txt index 057ebf219..5b5052aaa 100644 --- a/src/smith/physics/tests/CMakeLists.txt +++ b/src/smith/physics/tests/CMakeLists.txt @@ -50,6 +50,15 @@ if (TRIBOL_FOUND) ${physics_serial_tribol_test_sources}) endif() +if (SMITH_USE_DFEM) + list(APPEND physics_serial_test_sources + test_dfem_explicit_dynamics.cpp + test_dfem_lumped_mass.cpp + test_dfem_plasticity.cpp + test_dfem_vs_functional.cpp + test_dfem_weak_form.cpp) +endif() + smith_add_tests(SOURCES ${physics_serial_test_sources} DEPENDS_ON ${physics_test_depends} NUM_MPI_TASKS 1) diff --git a/src/smith/physics/tests/test_dfem_explicit_dynamics.cpp b/src/smith/physics/tests/test_dfem_explicit_dynamics.cpp new file mode 100644 index 000000000..c168ab612 --- /dev/null +++ b/src/smith/physics/tests/test_dfem_explicit_dynamics.cpp @@ -0,0 +1,277 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Smith Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +#include + +#include "mfem.hpp" +#include "smith/infrastructure/application_manager.hpp" +#include "smith/physics/mesh.hpp" +#include "smith/physics/state/state_manager.hpp" + +#include "smith/physics/functional_weak_form.hpp" +#include "smith/physics/dfem_solid_weak_form.hpp" +#include "smith/physics/dfem_mass_weak_form.hpp" + +auto element_shape = mfem::Element::QUADRILATERAL; + +const std::string MESHTAG = "mesh"; + +namespace mfem { +namespace future { + +/** + * @brief Compute Green's strain from the displacement gradient + */ +template +MFEM_HOST_DEVICE auto greenStrain(const tensor& grad_u) +{ + return 0.5 * (grad_u + transpose(grad_u) + dot(transpose(grad_u), grad_u)); +} + +} // namespace future +} // namespace mfem + +namespace smith { + +// NOTE (EBC): NeoHookean is not working with dfem on device with HIP, since some needed LLVM intrinsics are not +// implemented in Enzyme with the call to log1p()/log(). +struct StVenantKirchhoffWithFieldDensityDfem { + static constexpr int dim = 2; + + /** + * @brief stress calculation for a St. Venant Kirchhoff material model + * + * @tparam T Type of the displacement gradient components (number-like) + * + * @param[in] grad_u Displacement gradient + * + * @return The first Piola stress + */ + template + SMITH_HOST_DEVICE auto pkStress(double, const mfem::future::tensor& du_dX, + const mfem::future::tensor&, const Density&) const + { + auto I = mfem::future::IdentityMatrix(); + auto F = du_dX + I; + const auto E = mfem::future::greenStrain(du_dX); + + // stress + const auto S = K * mfem::future::tr(E) * I + 2.0 * G * mfem::future::dev(E); + return mfem::future::tuple{mfem::future::dot(F, S)}; + } + + /// @brief interpolates density field + template + SMITH_HOST_DEVICE auto density(const Density& density) const + { + return density; + } + + double K; ///< Bulk modulus + double G; ///< Shear modulus +}; + +} // namespace smith + +struct ExplicitDynamicsFixture : public testing::Test { + static constexpr int dim = 2; + static constexpr int disp_order = 1; + + using VectorSpace = smith::H1; + using DensitySpace = smith::L2; + + using SolidMaterialDfem = smith::StVenantKirchhoffWithFieldDensityDfem; + + enum STATE + { + DISPLACEMENT, + VELOCITY, + ACCELERATION, + COORDINATES + }; + + enum PARAMS + { + DENSITY + }; + + void SetUp() + { + MPI_Barrier(MPI_COMM_WORLD); + smith::StateManager::initialize(datastore, "solid_dynamics"); + + // create mesh + constexpr double length = 1.0; + constexpr double width = 1.0; + constexpr int nel_x = 4; + constexpr int nel_y = 5; + mesh = std::make_shared(mfem::Mesh::MakeCartesian2D(nel_x, nel_y, element_shape, true, length, width), + MESHTAG, 0, 0); + // shift one of the x coordinates so the mesh is not affine + auto* coords = mesh->mfemParMesh().GetNodes()->ReadWrite(); + coords[6] += 0.01; + + // create residual evaluator + smith::FiniteElementState disp = smith::StateManager::newState(VectorSpace{}, "displacement", mesh->tag()); + smith::FiniteElementState velo = smith::StateManager::newState(VectorSpace{}, "velocity", mesh->tag()); + smith::FiniteElementState accel = smith::StateManager::newState(VectorSpace{}, "acceleration", mesh->tag()); + smith::FiniteElementState density = smith::StateManager::newState(DensitySpace{}, "density", mesh->tag()); + disp.UseDevice(true); + velo.UseDevice(true); + accel.UseDevice(true); + density.UseDevice(true); + + states = {disp, velo, accel}; + params = {density}; + + std::string physics_name = "solid"; + double E = 1.0e3; + double nu = 0.3; + + using SolidT = smith::DfemSolidWeakForm; + auto solid_dfem_weak_form = + std::make_shared(physics_name, mesh, states[DISPLACEMENT].space(), getSpaces(params)); + + SolidMaterialDfem dfem_mat; + dfem_mat.K = E / (3.0 * (1.0 - 2.0 * nu)); // bulk modulus + dfem_mat.G = E / (2.0 * (1.0 + nu)); // shear modulus + int ir_order = 2; + const mfem::IntegrationRule& displacement_ir = mfem::IntRules.Get(disp.space().GetFE(0)->GetGeomType(), ir_order); + mfem::Array solid_attrib({1}); + solid_dfem_weak_form->setMaterial>(solid_attrib, dfem_mat, + displacement_ir); + + mfem::future::tensor g({0.0, -9.81}); // gravity vector + mfem::future::tuple, mfem::future::Value, + mfem::future::Value, mfem::future::Gradient, + mfem::future::Weight, mfem::future::Value> + g_inputs{}; + mfem::future::tuple> g_outputs{}; + solid_dfem_weak_form->addBodyIntegral( + solid_attrib, + [=] SMITH_HOST_DEVICE(const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor&, + const mfem::future::tensor& dX_dxi, mfem::real_t weight, double) { + auto J = mfem::future::det(dX_dxi) * weight; + return mfem::future::tuple{g * J}; + }, + g_inputs, g_outputs, displacement_ir, std::index_sequence<>{}); + + states[DISPLACEMENT] = 0.0; + states[VELOCITY] = 0.0; + states[ACCELERATION].setFromFieldFunction([](smith::tensor) { + smith::tensor u({0.0, -9.81}); + return u; + }); + params[DENSITY] = 1.0; + + dfem_weak_form = solid_dfem_weak_form; + + auto mass_dfem_weak_form = + smith::create_solid_mass_weak_form(physics_name, mesh, states[DISPLACEMENT], params[0], + displacement_ir); // nodal_ir_2d); + mass_weak_form = mass_dfem_weak_form; + + // create time advancer + advancer = std::make_shared(dfem_weak_form, mass_weak_form, nullptr); + } + + static constexpr bool quasi_static = true; + static constexpr bool lumped_mass = false; + + const double dt = 0.001; + const size_t num_steps = 3; + + // NOTE: max_error is driven by integration error on the perturbed element + static constexpr double max_error = 1.0e-12; + + axom::sidre::DataStore datastore; + std::shared_ptr mesh; + std::shared_ptr> dfem_weak_form; + + std::shared_ptr mass_weak_form; + + std::vector states; + std::vector params; + + mfem::IntegrationRule nodal_ir_2d; + std::shared_ptr advancer; +}; + +TEST_F(ExplicitDynamicsFixture, RunDfemExplicitDynamicsSim) +{ + // acceleration (constant) + mfem::Vector exact_accel_value({0.0, -9.81}); + smith::FiniteElementState exact_accel(states[ACCELERATION].space(), "exact_acceleration"); + exact_accel.UseDevice(true); + mfem::VectorConstantCoefficient exact_accel_coeff(exact_accel_value); + exact_accel.project(exact_accel_coeff); + auto exact_accel_ptr = exact_accel.HostRead(); + + // velocity + mfem::Vector exact_velo_value({0.0, exact_accel_value[1] * dt}); + + // displacement + mfem::Vector exact_disp_value({0.0, 0.5 * exact_velo_value[1] * dt}); + + double time = 0.0; + for (size_t step = 0; step < num_steps; ++step) { + for (auto& state : states) { + state.gridFunction(); + } + std::cout << "Step " << step << ", time = " << time << std::endl; + auto state_ptrs = smith::getConstFieldPointers(states); + smith::FiniteElementState coords(*static_cast(mesh->mfemParMesh().GetNodes())->ParFESpace(), + "coordinates"); + coords.UseDevice(true); + coords.setFromGridFunction(*static_cast(mesh->mfemParMesh().GetNodes())); + state_ptrs.push_back(&coords); + auto new_states_and_time = advancer->advanceState(state_ptrs, getConstFieldPointers(params), time, dt); + + time = std::get<1>(new_states_and_time); + for (size_t i = 0; i < states.size(); ++i) { + states[i] = std::get<0>(new_states_and_time)[i]; + } + + // check acceleration + auto accel_ptr = states[ACCELERATION].HostRead(); + for (int i = 0; i < states[ACCELERATION].Size(); ++i) { + EXPECT_NEAR(accel_ptr[i], exact_accel_ptr[i], max_error); + } + + // update and check velocity + smith::FiniteElementState exact_velo(states[VELOCITY].space(), "exact_velocity"); + exact_velo.UseDevice(true); + mfem::VectorConstantCoefficient exact_velo_coeff(exact_velo_value); + exact_velo.project(exact_velo_coeff); + auto velo_ptr = states[VELOCITY].HostRead(); + auto exact_velo_ptr = exact_velo.HostRead(); + for (int i = 0; i < states[VELOCITY].Size(); ++i) { + EXPECT_NEAR(velo_ptr[i], exact_velo_ptr[i], max_error); + } + exact_velo_value[1] += exact_accel_value[1] * dt; + + // update and check displacement + smith::FiniteElementState exact_disp(states[DISPLACEMENT].space(), "exact_displacement"); + exact_disp.UseDevice(true); + mfem::VectorConstantCoefficient exact_disp_coeff(exact_disp_value); + exact_disp.project(exact_disp_coeff); + auto disp_ptr = states[DISPLACEMENT].HostRead(); + auto exact_disp_ptr = exact_disp.HostRead(); + for (int i = 0; i < states[DISPLACEMENT].Size(); ++i) { + EXPECT_NEAR(disp_ptr[i], exact_disp_ptr[i], max_error); + } + exact_disp_value[1] += exact_velo_value[1] * dt - 0.5 * exact_accel_value[1] * dt * dt; + } +} + +int main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + smith::ApplicationManager applicationManager(argc, argv, MPI_COMM_WORLD, true, smith::ExecutionSpace::GPU); + return RUN_ALL_TESTS(); +} diff --git a/src/smith/physics/tests/test_dfem_lumped_mass.cpp b/src/smith/physics/tests/test_dfem_lumped_mass.cpp new file mode 100644 index 000000000..856abea66 --- /dev/null +++ b/src/smith/physics/tests/test_dfem_lumped_mass.cpp @@ -0,0 +1,308 @@ +// Copyright (c) Lawrence Livermore National Security, LLC and +// other Smith Project Developers. See the top-level LICENSE file for +// details. +// +// SPDX-License-Identifier: (BSD-3-Clause) + +#include +#include "mfem.hpp" +#include "smith/infrastructure/application_manager.hpp" +#include "smith/physics/mesh.hpp" +#include "smith/physics/state/state_manager.hpp" + +#include "smith/physics/tests/physics_test_utils.hpp" +#include "smith/physics/dfem_solid_weak_form.hpp" +#include "smith/physics/dfem_mass_weak_form.hpp" +#include "smith/physics/functional_weak_form.hpp" + +auto element_shape = mfem::Element::QUADRILATERAL; + +namespace smith { + +template