diff --git a/.DS_Store b/.DS_Store
deleted file mode 100644
index c6ec70fbb..000000000
Binary files a/.DS_Store and /dev/null differ
diff --git a/.gitignore b/.gitignore
index cb3b80d30..7d690806e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,3 +54,4 @@ doc/*
!readme
!figures/*
+.DS_Store
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2daf2a749..3bc788f49 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,102 +1,40 @@
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.15)
project(m3dc1 LANGUAGES Fortran C CXX)
-# macro(checkSetParam varName isRequired)
-# if("${${varName}}" STREQUAL "") # if varName is empty
-# set(${varName} $ENV{${varName}})
-# if(("${${varName}}" STREQUAL "") AND ${isRequired}) # if ENV{varName} is empty also
-# message(FATAL_ERROR
-# "${varName} is not defined or in the environment, please specify ${varName}")
-# endif()
-# endif()
-# endmacro(checkSetParam varName)
-
-option(ENABLE_OPENMP "Enable OpenMP support" OFF)
-option(ENABLE_COMPLEX "Build complex version" OFF)
-option(ENABLE_3D "Build 3D version" OFF)
-option(ENABLE_ST "Build stellarator version " OFF)
-option(ENABLE_PARTICLE "Enable kinetic particle module" OFF)
+option(M3DC1_ENABLE_OPENMP "Enable OpenMP support" OFF)
+if(M3DC1_ENABLE_OPENMP)
+ find_package(OpenMP REQUIRED COMPONENTS Fortran)
+endif()
+option(M3DC1_ENABLE_COMPLEX "Build complex version" OFF)
+option(M3DC1_ENABLE_3D "Build 3D version" OFF)
+option(M3DC1_ENABLE_ST "Build stellarator version " OFF)
+option(M3DC1_ENABLE_PARTICLE "Enable kinetic particle module" OFF)
+option(M3DC1_ENABLE_GPU "Use GPU/OpenACC Fortran kernels" OFF)
+option(M3DC1_ENABLE_ADAS "Enable ADAS radiation data support" OFF)
+option(M3DC1_ENABLE_TRILINOS "Enable Trilinos support" OFF)
+option(M3DC1_ENABLE_PSPLINE "Enable PSPLINE support for readgato/read_jsolver" OFF)
+option(M3DC1_DOWNLOAD_ADAS_DATA "Download OpenADAS ADF11 data as part of the default build" OFF)
+set(M3DC1_ADAS_DATA_DIR "${CMAKE_BINARY_DIR}/adas/adf11"
+ CACHE PATH "Path to OpenADAS ADF11 data directory")
cmake_host_system_information(RESULT HOST QUERY HOSTNAME)
message(STATUS "Configuring to build on: ${HOST}")
-# # rhel/centos place x86-64 libs in lib64 instead of lib
-# # this *shouldn't* effect searching the lib directory
-# # first on other distros where the default install location
-# # is lib instead of lib64 (like debian derivatives)
-# set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS TRUE)
-
-# prefer static libraries unless we explicitly specify no to
-# set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so")
-# if(BUILD_SHARED_LIBS)
-# set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a")
-# endif()
-
-# set the build type logically
- # default is Debug if .git directory exists
- # otherwise default is Release
-# list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-# include(BuildType)
-# # add C++ compiler flags
-# if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Cray")
-# # todo : set cray-specific cxx flags
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-# elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "XL")
-# # todo : check for regular xl vs bg/q xl compiler
-# # currently assuming bg/q
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qflag=w -glanglvl=extended0x")
-# elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
-# # todo : set intel-specific cxx flags
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-# elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-# # todo : version checks to see which c++ standards are available
-# # and warn/error if the version is incompatible
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x -Wall -Wextra")
-# elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-# # todo : version checks to see which c++ standards are available
-# # and warn/error if the version is incompatible
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++1z -Wall -Wextra")
-# endif()
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake"
+ "${CMAKE_CURRENT_LIST_DIR}/m3dc1_scorec/cmake")
+find_package(PETSc REQUIRED)
-# # add C compiler flags
-# if("${CMAKE_C_COMPILER_ID}" STREQUAL "Cray")
-# # todo : set cray-specific c flags
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-# elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "XL")
-# # todo : set xl-specific c flags
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-# elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
-# # todo : set intel-specific c flags
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-# elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
-# # todo : version checks to see which c standards are available
-# # and warn/error if the version is incompatible
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
-# elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
-# # todo : version checks to see which c++ standards are available
-# # and warn/error if the version is incompatible
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra")
-# endif()
+# Automatically set exe linker flags from PETSc pkg-config data.
+# Construct -L
flags from PETSC_LIBRARY_DIRS.
+string(REPLACE ";" " -L" _m3dc1_petsc_libdirs_flags "${PETSC_LIBRARY_DIRS}")
+if(_m3dc1_petsc_libdirs_flags)
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L${_m3dc1_petsc_libdirs_flags}")
+endif()
+# Append PETSc's extra LDFLAGS (which include rpath from pkg-config).
+if(PETSC_LDFLAGS_OTHER)
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PETSC_LDFLAGS_OTHER}")
+endif()
-# # add Fortran compiler flags
-# if("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Cray")
-# # todo : set cray-specific fortran flags
-# set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
-# elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "XL")
-# # todo : set xl-specific fortran flags
-# set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}")
-# elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Intel")
-# # todo : set intel-specific fortran flags
-# set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -g noarg_temp_created")
-# elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "GNU")
-# set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -Wall -cpp")
-# elseif("${CMAKE_Fortran_COMPILER_ID}" STREQUAL "Clang")
-# set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -Wall -cpp")
-# endif()
-
-message(STATUS "Configuring m3dc1_scorec")
-add_subdirectory(m3dc1_scorec EXCLUDE_FROM_ALL)
-message(STATUS "Configuring unstructured")
+add_subdirectory(m3dc1_scorec)
add_subdirectory(unstructured)
-
diff --git a/cmake/DownloadOpenADAS.cmake b/cmake/DownloadOpenADAS.cmake
new file mode 100644
index 000000000..9b966bc27
--- /dev/null
+++ b/cmake/DownloadOpenADAS.cmake
@@ -0,0 +1,82 @@
+# Download OpenADAS ADF11 data and reader sources
+# Called from unstructured/CMakeLists.txt with:
+# -DOUT_DIR= : output directory (CMAKE_BINARY_DIR/adas)
+# -DADF11_DIR= : where ADF11 data files are placed
+
+set(OA_DATA_URL "https://open.adas.ac.uk/download/adf11")
+set(OA_SOURCE_DIR "${OUT_DIR}/source")
+
+# ---- Download and extract reader source code ----
+message(STATUS "Downloading OpenADAS reader sources...")
+file(DOWNLOAD
+ "https://open.adas.ac.uk/code/xxdata_11.tar.gz"
+ "${OUT_DIR}/xxdata_11.tar.gz"
+ STATUS _dl_status
+ SHOW_PROGRESS
+)
+list(GET _dl_status 0 _dl_code)
+if(NOT _dl_code EQUAL 0)
+ message(FATAL_ERROR "Failed to download xxdata_11.tar.gz")
+endif()
+
+# Extract into a temporary directory, then move .for → .f to source/
+set(_extract_dir "${OUT_DIR}/_extract_tmp")
+file(REMOVE_RECURSE "${_extract_dir}")
+file(MAKE_DIRECTORY "${_extract_dir}")
+file(MAKE_DIRECTORY "${OA_SOURCE_DIR}")
+
+execute_process(
+ COMMAND ${CMAKE_COMMAND} -E tar xzf "${OUT_DIR}/xxdata_11.tar.gz"
+ WORKING_DIRECTORY "${_extract_dir}"
+ RESULT_VARIABLE _tar_res
+)
+if(NOT _tar_res EQUAL 0)
+ message(FATAL_ERROR "Failed to extract xxdata_11.tar.gz")
+endif()
+
+# Find all .for files (recursively, handles possible subdirectories)
+# and copy/rename them to OA_SOURCE_DIR as .f
+file(GLOB_RECURSE _for_files "${_extract_dir}/*.for")
+foreach(_f IN LISTS _for_files)
+ get_filename_component(_name "${_f}" NAME_WE)
+ file(COPY "${_f}" DESTINATION "${OA_SOURCE_DIR}")
+ file(RENAME "${OA_SOURCE_DIR}/${_name}.for" "${OA_SOURCE_DIR}/${_name}.f")
+endforeach()
+
+# Cleanup
+file(REMOVE_RECURSE "${_extract_dir}")
+file(REMOVE "${OUT_DIR}/xxdata_11.tar.gz")
+message(STATUS "OpenADAS reader sources extracted to ${OA_SOURCE_DIR}")
+
+# ---- Download ADF11 data files ----
+set(_adf11_files
+ "scd85/scd85_ar.dat"
+ "scd89/scd89_b.dat"
+ "plt89/plt89_b.dat"
+ "plt89/plt89_ar.dat"
+ "scd96/scd96_he.dat"
+ "scd96/scd96_be.dat"
+ "scd96/scd96_c.dat"
+ "scd96/scd96_ne.dat"
+ "plt96/plt96_he.dat"
+ "plt96/plt96_be.dat"
+ "plt96/plt96_c.dat"
+ "plt96/plt96_ne.dat"
+)
+
+foreach(_f IN LISTS _adf11_files)
+ get_filename_component(_subdir "${_f}" DIRECTORY)
+ file(MAKE_DIRECTORY "${ADF11_DIR}/${_subdir}")
+ message(STATUS "Downloading ${_f}...")
+ file(DOWNLOAD
+ "${OA_DATA_URL}/${_f}"
+ "${ADF11_DIR}/${_f}"
+ STATUS _dl_status
+ )
+ list(GET _dl_status 0 _dl_code)
+ if(NOT _dl_code EQUAL 0)
+ message(FATAL_ERROR "Failed to download ${_f}")
+ endif()
+endforeach()
+
+message(STATUS "OpenADAS download complete")
\ No newline at end of file
diff --git a/m3dc1_scorec/CMakeLists.txt b/m3dc1_scorec/CMakeLists.txt
index b1b717a0d..b3539b6d6 100644
--- a/m3dc1_scorec/CMakeLists.txt
+++ b/m3dc1_scorec/CMakeLists.txt
@@ -1,62 +1,3 @@
-#This is the top M3DC1_SCOREC CMakeList File for the Build
-
-#Setting Version Number, Project Name
-cmake_minimum_required (VERSION 2.8)
-project (m3dc1_scorec)
-enable_language (Fortran)
-
-# make sure that the default is a RELEASE
-if (NOT CMAKE_BUILD_TYPE)
- set (CMAKE_BUILD_TYPE RELEASE CACHE STRING
- "Choose the type of build, options are: None Debug Release."
- FORCE)
-endif (NOT CMAKE_BUILD_TYPE)
-
-# default installation
-#get_filename_component (default_prefix ".." ABSOLUTE)
-#set (CMAKE_INSTALL_PREFIX ${default_prefix} CACHE STRING
-# "Choose the installation directory; by default it installs in the NORMA directory."
-# FORCE)
-
-# FFLAGS depend on the compiler
-get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
-message (Fortran_COMPILER_NAME = ${Fortran_COMPILER_NAME})
-
-#unless building shared libs, then select static libs
-# if both static and shared libs are available
-set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so")
-if(BUILD_SHARED_LIBS)
- set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a")
-endif()
-
-#Settings options for testing
-enable_testing()
-include(CTest)
-#This will be set to ON by the CTest driver script (and only by that)
-option(ENABLE_TESTING "Build for CTest" OFF)
-set(MPIRUN "mpirun"
- CACHE string
- "the mpirun or srun executable")
-set(MPIRUN_PROCFLAG "-np"
- CACHE string
- "the command line flag to give process count to MPIRUN")
-
-#Doxygen generation system
-find_package(Doxygen)
-if(DOXYGEN_FOUND)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in
- ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile @ONLY)
-add_custom_target(doc
-${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile
-WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-COMMENT "Generating API documentation with Doxygen" VERBATIM
-)
-endif(DOXYGEN_FOUND)
-
-configure_file(
- "${CMAKE_CURRENT_SOURCE_DIR}/m3dc1_scorecConfig.cmake.in"
- "${CMAKE_BINARY_DIR}/m3dc1_scorecConfig.cmake")
-
#Source and header files
set(SOURCES
api/m3dc1_scorec.cc
@@ -97,143 +38,37 @@ set(HEADERS
#include/ReducedQuinticExplicit.h
include/slntransferUtil.h
include/Expression.h
-
include/ReducedQuintic.h)
-set(CMAKE_MODULE_PATH
- ${CMAKE_MODULE_PATH}
- "${CMAKE_CURRENT_SOURCE_DIR}/cmake/")
-
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
-include_directories(${CMAKE_CURRENT_SOURCE_DIR}/api)
-set(TARGET_LIB_NAME m3dc1_scorec)
-
-find_package(Scorec QUIET REQUIRED)
-include_directories(${SCOREC_INCLUDE_DIRS})
-set(DEP_LIBS ${DEP_LIBS} ${SCOREC_LIBRARIES} )
-if (ENABLE_SIMMETRIX)
-set(ENABLE_PETSC "OFF")
-set(ENABLE_TRILINOS "OFF")
-find_package(Simmetrix QUIET REQUIRED)
-include_directories(${SIMMETRIX_INCLUDE_DIRS})
-set(DEP_LIBS ${DEP_LIBS} ${SIMMETRIX_LIBRARIES} )
-set(TARGET_LIB_NAME ${TARGET_LIB_NAME}_simmetrix)
-find_package(Lapack QUIET REQUIRED)
-set(DEP_LIBS ${DEP_LIBS} ${LAPACK_LIBRARIES} )
- add_executable(m3dc1_meshgen test/meshgen/m3dc1_meshgen.cc)
- target_link_libraries(m3dc1_meshgen ${DEP_LIBS})
- target_link_libraries(m3dc1_meshgen ${TARGET_LIB_NAME})
- INSTALL(TARGETS m3dc1_meshgen RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
- add_executable(convert_sim_sms test/meshgen/convert_sim_sms.cc)
- target_link_libraries(convert_sim_sms ${DEP_LIBS})
- target_link_libraries(convert_sim_sms ${TARGET_LIB_NAME})
- INSTALL(TARGETS convert_sim_sms RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
- add_executable(convert_polar test/convert_polar/main.cc)
- target_link_libraries(convert_polar ${DEP_LIBS})
- target_link_libraries(convert_polar ${TARGET_LIB_NAME})
- INSTALL(TARGETS convert_polar RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
-elseif (ENABLE_TRILINOS)
-find_package(Trilinos QUIET REQUIRED)
-find_package(Boost QUIET REQUIRED)
-include_directories(${TRILINOS_INCLUDE_DIRS})
-set(DEP_LIBS ${DEP_LIBS} ${TRILINOS_LIBRARIES})
-set(DEP_LIBS ${DEP_LIBS} ${BOOST_LIBRARIES})
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DM3DC1_TRILINOS")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DM3DC1_TRILINOS")
-set(TARGET_LIB_NAME ${TARGET_LIB_NAME}_trilinos)
-else()
- set(ENABLE_PETSC "ON")
-endif()
+find_package(SCOREC CONFIG REQUIRED)
+find_package(LAPACK REQUIRED)
+find_package(MPI REQUIRED COMPONENTS C)
-option(ENABLE_ZOLTAN "Enable Zoltan interface [ON|OFF]" ON)
-message(STATUS "ENABLE_ZOLTAN: " ${ENABLE_ZOLTAN})
-
-if (ENABLE_ZOLTAN)
-find_library(ZOLTAN_LIBRARY zoltan)
-if (NOT EXISTS "${ZOLTAN_LIBRARY}")
- message(FATAL ERROR "ZOLTAN library not found")
-endif()
-else()
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DDISABLE_ZOLTAN")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DDISABLE_ZOLTAN")
-endif()
-
-if (ENABLE_PETSC)
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DM3DC1_PETSC")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DM3DC1_PETSC")
-#find_package(Hdf5 QUIET REQUIRED)
-#set(DEP_LIBS ${DEP_LIBS} ${HDF5_LIBRARIES} )
-find_package(Petsc QUIET REQUIRED)
-include_directories(${PETSC_INCLUDE_DIRS})
-set(DEP_LIBS ${DEP_LIBS} ${PETSC_LIBRARIES} )
-endif()
-
-if(ENABLE_COMPLEX)
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DPETSC_USE_COMPLEX")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPETSC_USE_COMPLEX")
-set(TARGET_LIB_NAME ${TARGET_LIB_NAME}_complex)
-endif()
-
-# if(ENABLE_OPENMP)
-# if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU")
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fopenmp")
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp")
-# elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL "Intel")
-# set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qopenmp")
-# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -qopenmp")
-# else()
-# message(WARNING "Unexpected compiler type ${CMAKE_CXX_COMPILER_ID}")
-# endif()
-# set(TARGET_LIB_NAME ${TARGET_LIB_NAME}_openmp)
-# endif()
-
-add_library(${TARGET_LIB_NAME} ${SOURCES})
-target_link_libraries(${TARGET_LIB_NAME} ${DEP_LIBS})
-
-INSTALL(FILES ${HEADERS} DESTINATION include)
-INSTALL(TARGETS ${TARGET_LIB_NAME}
- LIBRARY DESTINATION lib
- ARCHIVE DESTINATION lib)
-
-if (ENABLE_TESTING)
-if (ENABLE_PETSC)
- add_executable(solver test/solver/main.cc)
- target_link_libraries(solver ${DEP_LIBS})
- target_link_libraries(solver ${TARGET_LIB_NAME})
- INSTALL(TARGETS solver RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
-endif()
-
-if (ENABLE_TRILINOS)
- add_executable(epetra test/epetra/main.cc)
- target_link_libraries(epetra ${DEP_LIBS})
- target_link_libraries(epetra ${TARGET_LIB_NAME})
- INSTALL(TARGETS epetra RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
-endif()
-add_executable(show_meshcount test/create_smb/show_meshcount.cc)
-target_link_libraries(show_meshcount ${DEP_LIBS})
-target_link_libraries(show_meshcount ${TARGET_LIB_NAME})
-INSTALL(TARGETS show_meshcount RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
-endif()
-
-#add_executable(ghosting_test test/ghosting/main.cc)
-#target_link_libraries(ghosting_test ${DEP_LIBS})
-#target_link_libraries(ghosting_test ${TARGET_LIB_NAME})
-#INSTALL(TARGETS ghosting_test RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin)
-
-#binary distribution package
-set(CPACK_GENERATOR "TGZ")
-set(CPACK_PACKAGE_VERSION "1.0.1")
-include(CPack)
-
-message("ENV_FLAGS = $ENV{FFLAGS}")
-message("CMAKE_C_FLAGS = ${CMAKE_C_FLAGS}")
-message("CMAKE_CXX_FLAGS = ${CMAKE_CXX_FLAGS}")
-message("CMAKE_Fortran_COMPILER_INIT = ${CMAKE_Fortran_COMPILER_INIT}")
-message("CMAKE_Fortran_COMPILER_FULLPATH =
-${CMAKE_Fortran_COMPILER_FULLPATH}")
-message("CMAKE_Fortran_COMPILER = ${CMAKE_Fortran_COMPILER}")
-message("CMAKE_Fortran_FLAGS = ${CMAKE_Fortran_FLAGS}")
-message("CMAKE_INSTALL_PREFIX = ${CMAKE_INSTALL_PREFIX}")
+add_library(m3dc1_scorec ${SOURCES})
+add_library(m3dc1::scorec ALIAS m3dc1_scorec)
+target_link_libraries(m3dc1_scorec PUBLIC SCOREC::core PETSc::PETSc LAPACK::LAPACK MPI::MPI_C)
+target_include_directories(m3dc1_scorec PUBLIC
+ "${CMAKE_CURRENT_SOURCE_DIR}/api"
+ "${CMAKE_CURRENT_SOURCE_DIR}/include"
+)
+target_compile_definitions(m3dc1_scorec PRIVATE M3DC1_PETSC)
+
+# --- Utility executables (SCOREC tools) ---
+# Utilities that only depend on SCOREC core
+set(M3DC1_UTILITY_CORE_SOURCES
+ utility/check_smb.cc
+ utility/create_smb.cc
+ utility/split_smb.cc
+ utility/vtk_order.cc
+)
+foreach(_util_src IN LISTS M3DC1_UTILITY_CORE_SOURCES)
+ get_filename_component(_util_name ${_util_src} NAME_WE)
+ add_executable(${_util_name} ${_util_src})
+ target_link_libraries(${_util_name} PRIVATE SCOREC::core MPI::MPI_C)
+endforeach()
+# show_meshcount depends on the m3dc1_scorec library (m3dc1_mesh, m3dc1_model, etc.)
+add_executable(show_meshcount utility/show_meshcount.cc)
+target_link_libraries(show_meshcount PRIVATE m3dc1_scorec)
diff --git a/m3dc1_scorec/cmake/FindPETSc.cmake b/m3dc1_scorec/cmake/FindPETSc.cmake
new file mode 100644
index 000000000..24b1e0f70
--- /dev/null
+++ b/m3dc1_scorec/cmake/FindPETSc.cmake
@@ -0,0 +1,54 @@
+find_package(PkgConfig REQUIRED QUIET)
+if(DEFINED PETSC_DIR)
+ if(DEFINED PETSC_ARCH)
+ set(ENV{PKG_CONFIG_PATH} "${PETSC_DIR}/${PETSC_ARCH}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}")
+ else()
+ set(ENV{PKG_CONFIG_PATH} "${PETSC_DIR}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}")
+ endif()
+endif()
+
+# we give an internal name _petsc
+# so we can fill up the PETSC_VARIABLE based
+# on static or not
+pkg_check_modules(_petsc PETSc QUIET)
+
+if(_petsc_FOUND AND _petsc_VERSION)
+ set(PETSC_VERSION ${_petsc_VERSION})
+endif()
+
+# note, there are a number of additional properties that
+# can be extracted / set on a target. We just do the basic
+# set for now. See: https://cmake.org/cmake/help/latest/module/FindPkgConfig.html
+if(PETSC_LINK_STATIC)
+ set(PETSC_LIBRARIES ${_petsc_STATIC_LIBRARIES})
+ set(PETSC_INCLUDE_DIRS ${_petsc_STATIC_INCLUDE_DIRS})
+ set(PETSC_LIBRARY_DIRS ${_petsc_STATIC_LIBRARY_DIRS})
+ set(PETSC_LDFLAGS ${_petsc_STATIC_LDFLAGS})
+ set(PETSC_LDFLAGS_OTHER ${_petsc_STATIC_LDFLAGS_OTHER})
+elseif(_petsc_FOUND)
+ set(PETSC_LIBRARIES ${_petsc_LIBRARIES})
+ set(PETSC_INCLUDE_DIRS ${_petsc_INCLUDE_DIRS})
+ set(PETSC_LIBRARY_DIRS ${_petsc_LIBRARY_DIRS})
+ set(PETSC_LDFLAGS ${_petsc_LDFLAGS})
+ set(PETSC_LDFLAGS_OTHER ${_petsc_LDFLAGS_OTHER})
+endif()
+
+if(NOT TARGET PETSc::PETSc)
+ add_library(PETSc::PETSc INTERFACE IMPORTED GLOBAL)
+ set_target_properties(PETSc::PETSc PROPERTIES INTERFACE_LINK_LIBRARIES "${PETSC_LIBRARIES}"
+ INTERFACE_INCLUDE_DIRECTORIES "${PETSC_INCLUDE_DIRS}"
+ INTERFACE_LINK_DIRECTORIES "${PETSC_LIBRARY_DIRS}"
+ INTERFACE_LINK_OPTIONS "${PETSC_LDFLAGS_OTHER}")
+endif()
+
+
+
+include(FindPackageHandleStandardArgs)
+# TODO consider adding version check logic
+find_package_handle_standard_args(PETSc
+ REQUIRED_VARS PETSC_LIBRARIES PETSC_INCLUDE_DIRS
+ VERSION_VAR PETSC_VERSION)
+
+#message(STATUS "STATIC_LIBRARIES=${_petsc_STATIC_LIBRARIES}")
+#message(STATUS "STATIC_LDFLAGS=${_petsc_STATIC_LDFLAGS}")
+#message(STATUS "STATIC_LDFLAGS_OTHER=${_petsc_STATIC_LDFLAGS_OTHER}")
\ No newline at end of file
diff --git a/m3dc1_scorec/cmake/FindScorec.cmake b/m3dc1_scorec/cmake/FindScorec.cmake
deleted file mode 100644
index 9ac49bc75..000000000
--- a/m3dc1_scorec/cmake/FindScorec.cmake
+++ /dev/null
@@ -1,114 +0,0 @@
-# - Try to find SCOREC PUMI libraries
-# Once done this will define
-# SCOREC_FOUND - System has SCOREC
-# SCOREC_INCLUDE_DIRS - The SCOREC include directories
-# SCOREC_LIBRARIES - The libraries needed to use SCOREC
-# SCOREC_DEFINITIONS - Compiler switches required for using SCOREC
-#
-# This implementation assumes a SCOREC install has the following structure
-# VERSION/
-# include/*.h
-# lib/*.a
-
-macro(scorecLibCheck libs isRequired)
- foreach(lib ${libs})
- unset(scoreclib CACHE)
- find_library(scoreclib "${lib}" PATHS ${SCOREC_LIB_DIR})
- if(scoreclib MATCHES "^scoreclib-NOTFOUND$")
- if(${isRequired})
- message(FATAL_ERROR "SCOREC library ${lib} not found in ${SCOREC_LIB_DIR}")
- else()
- message("SCOREC library ${lib} not found in ${SCOREC_LIB_DIR}")
- endif()
- else()
- set("SCOREC_${lib}_FOUND" TRUE CACHE INTERNAL "SCOREC library present")
- set(SCOREC_LIBS ${SCOREC_LIBS} ${scoreclib})
- endif()
- endforeach()
-endmacro(scorecLibCheck)
-
-if(CMAKE_CXX_COMPILER_ID MATCHES "PGI")
- set(ENABLE_ZOLTAN OFF)
-endif()
-
-find_library(METIS_LIBRARY metis)
-if (NOT EXISTS "${METIS_LIBRARY}")
- message(FATAL ERROR "METIS library not found")
-endif()
-
-set(SCOREC_LIBS "")
-if(ENABLE_SIMMETRIX)
-set(SCOREC_LIB_NAMES
- ma
- mds
- apf_sim
- apf_zoltan
- apf
- parma
- gmi
- gmi_sim
- mth
- pcu
- sam
- spr
- crv
- lion
- ph
- )
-else()
-set(SCOREC_LIB_NAMES
-crv
-ph
-sam
-spr
-ma
-apf_zoltan
-parma
-mds
-apf
-lion
-mth
-gmi
-pcu
- )
-endif()
-scorecLibCheck("${SCOREC_LIB_NAMES}" TRUE)
-
-find_path(SCOREC_INCLUDE_DIR
- NAMES apf.h PCU.h ma.h
- PATHS ${SCOREC_INCLUDE_DIR})
-if(NOT EXISTS "${SCOREC_INCLUDE_DIR}")
- message(FATAL_ERROR "SCOREC include dir not found")
-endif()
-
-string(REGEX REPLACE
- "/include$" ""
- SCOREC_INSTALL_DIR
- "${SCOREC_INCLUDE_DIR}")
-
-set(SCOREC_LIBRARIES ${SCOREC_LIBS} ${ZOLTAN_LIBRARY} ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
-set(SCOREC_INCLUDE_DIRS ${SCOREC_INCLUDE_DIR})
-
-include(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set PARMETIS_FOUND to TRUE
-# if all listed variables are TRUE
-find_package_handle_standard_args(SCOREC DEFAULT_MSG
- SCOREC_LIBS SCOREC_INCLUDE_DIR)
-
-mark_as_advanced(SCOREC_INCLUDE_DIR SCOREC_LIBS ${ZOLTAN_LIBRARY} ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
-
-set(SCOREC_LINK_LIBS "")
-foreach(lib ${SCOREC_LIB_NAMES})
- set(SCOREC_LINK_LIBS "${SCOREC_LINK_LIBS} -l${lib}")
-endforeach()
-
-#pkgconfig
-set(prefix "${SCOREC_INSTALL_DIR}")
-set(includedir "${SCOREC_INCLUDE_DIR}")
-configure_file(
- "cmake/libScorec.pc.in"
- "${CMAKE_BINARY_DIR}/libScorec.pc"
- @ONLY)
-
-INSTALL(FILES "${CMAKE_BINARY_DIR}/libScorec.pc" DESTINATION lib/pkgconfig)
-
diff --git a/m3dc1_scorec/include/ReducedQuinticExplicit.h b/m3dc1_scorec/include/ReducedQuinticExplicit.h
new file mode 100644
index 000000000..78ca38db1
--- /dev/null
+++ b/m3dc1_scorec/include/ReducedQuinticExplicit.h
@@ -0,0 +1,32 @@
+/******************************************************************************
+
+ (c) 2005-2017 Scientific Computation Research Center,
+ Rensselaer Polytechnic Institute. All rights reserved.
+
+ This work is open source software, licensed under the terms of the
+ BSD license as described in the LICENSE file in the top-level directory.
+
+*******************************************************************************/
+#ifndef REDUCEDQUINTICEXPLICIT_H
+#define REDUCEDQUINTICEXPLICIT_H
+#include "ReducedQuintic.h"
+// see Barnhill 1981
+class ReducedQuinticExplicit : public ReducedQuintic
+{
+public:
+ virtual void setCoord(double coord[3][2]);
+ // dofs in glb coord
+ virtual void setDofs(double dofs_p[18]);
+ virtual void print();
+ virtual void eval_l( double coord[2], double res[6] );
+ virtual void eval_g( double coord[2], double res[6] );
+ virtual void loc2glb( double coord[2]);
+ virtual void glb2loc( double coord[2]);
+ virtual void loc2glbDofs(double dofs_p[18]);
+ virtual void glb2locDofs(double dofs_p[18]);
+ virtual double getJacobi();
+private:
+ double dofs[18];
+};
+
+#endif
diff --git a/m3dc1_scorec/src/ReducedQuinticExplicit.cc b/m3dc1_scorec/src/ReducedQuinticExplicit.cc
new file mode 100644
index 000000000..c66b2cde9
--- /dev/null
+++ b/m3dc1_scorec/src/ReducedQuinticExplicit.cc
@@ -0,0 +1,86 @@
+/******************************************************************************
+
+ (c) 2005-2017 Scientific Computation Research Center,
+ Rensselaer Polytechnic Institute. All rights reserved.
+
+ This work is open source software, licensed under the terms of the
+ BSD license as described in the LICENSE file in the top-level directory.
+
+*******************************************************************************/
+#include "ReducedQuinticExplicit.h"
+extern "C"
+{
+ void set_element_coord_i_ ( int*, double*);
+ void eval_field_i_( double *, double*, double*, double*);
+ void loc_xyz_2_glb_i_ ( double*, double*);
+ void glb_xyz_2_loc_i_ ( double*, double*);
+ void glb_dof_2_loc_i_ ( double*, double*);
+ void loc_dof_2_glb_i_ ( double*, double*);
+ void get_jacobi_i_( double *);
+}
+double ReducedQuinticExplicit:: getJacobi()
+{
+ double jac;
+ get_jacobi_i_(&jac);
+ return jac;
+}
+void ReducedQuinticExplicit:: setCoord(double coord[3][2])
+{
+ double coord_f[2][3];
+ // convert to fortran convention
+ for(int i=0; i<3; i++)
+ {
+ for( int j=0; j<2; j++)
+ coord_f[j][i]=coord[i][j];
+ }
+ int dummy=0;
+ set_element_coord_i_(&dummy,(double*)coord_f);
+}
+void ReducedQuinticExplicit:: setDofs(double dofs_p[18])
+{
+ for(int i=0;i<18; i++)
+ dofs[i]=dofs_p[i];
+ glb2locDofs(dofs);
+}
+void ReducedQuinticExplicit:: print()
+{
+}
+void ReducedQuinticExplicit:: eval_g( double coord[2], double res[6] )
+{
+ double coord_loc[2]={coord[0],coord[1]};
+ glb2loc(coord_loc);
+ eval_field_i_(coord_loc,coord_loc+1,dofs,res);
+}
+void ReducedQuinticExplicit:: eval_l( double coord[2], double res[6] )
+{
+ double coord_loc[2]={coord[0],coord[1]};
+ eval_field_i_(coord_loc,coord_loc+1,dofs,res);
+}
+void ReducedQuinticExplicit:: loc2glb( double coord[2])
+{
+ double coord_glb[2];
+ loc_xyz_2_glb_i_(coord,coord_glb);
+ coord[0]=coord_glb[0];
+ coord[1]=coord_glb[1];
+}
+void ReducedQuinticExplicit:: glb2loc( double coord[2])
+{
+ double coord_loc[2];
+ glb_xyz_2_loc_i_(coord,coord_loc);
+ coord[0]=coord_loc[0];
+ coord[1]=coord_loc[1];
+}
+void ReducedQuinticExplicit:: loc2glbDofs(double dofs_p[18])
+{
+ double dofs_glb[18];
+ loc_dof_2_glb_i_(dofs_p,dofs_glb);
+ for(int i=0; i<18; i++)
+ dofs_p[i]=dofs_glb[i];
+}
+void ReducedQuinticExplicit:: glb2locDofs(double dofs_p[18])
+{
+ double dofs_loc[18];
+ glb_dof_2_loc_i_(dofs_p,dofs_loc);
+ for(int i=0; i<18; i++)
+ dofs_p[i]=dofs_loc[i];
+}
diff --git a/m3dc1_scorec/utility/check_smb.cc b/m3dc1_scorec/utility/check_smb.cc
new file mode 100644
index 000000000..371f243ff
--- /dev/null
+++ b/m3dc1_scorec/utility/check_smb.cc
@@ -0,0 +1,128 @@
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#ifdef HAVE_SIMMETRIX
+#include
+#include
+#endif
+#include
+#include
+#include
+#include
+
+namespace {
+
+const char* meshFile = 0;
+
+void freeMesh(apf::Mesh* m)
+{
+ m->destroyNative();
+ apf::destroyMesh(m);
+}
+
+void getConfig(int argc, char** argv)
+{
+ if ( argc < 2 ) {
+ if ( !PCU_Comm_Self() )
+ printf("Usage: %s \n", argv[0]);
+ MPI_Finalize();
+ exit(EXIT_FAILURE);
+ }
+}
+
+void print_info (apf::Mesh2* m)
+{
+ if (!PCU_Comm_Self()) std::cout<<"\n===== mesh size =====\n";
+
+ int* local_entity_count = new int[4*PCU_Comm_Peers()];
+ int* own_entity_count = new int[4*PCU_Comm_Peers()];
+
+ for (int i=0; i<4*PCU_Comm_Peers();++i)
+ local_entity_count[i]=own_entity_count[i]=0;
+
+ apf::MeshEntity* e;
+ int self = PCU_Comm_Self();
+
+ for (int d=0; d<4;++d)
+ {
+ local_entity_count[4*self+d] = m->count(d);
+ apf::MeshIterator* it = m->begin(d);
+ while ((e = m->iterate(it)))
+ {
+ if (m->getOwner(e)==self)
+ ++own_entity_count[4*PCU_Comm_Self()+d];
+ }
+ m->end(it);
+ }
+
+ int* global_local_entity_count = new int[4*PCU_Comm_Peers()];
+ int* global_own_entity_count = new int[4*PCU_Comm_Peers()];
+
+ MPI_Allreduce(local_entity_count, global_local_entity_count, 4*PCU_Comm_Peers(),
+ MPI_INT, MPI_SUM, PCU_Get_Comm());
+
+ MPI_Allreduce(own_entity_count, global_own_entity_count, 4*PCU_Comm_Peers(),
+ MPI_INT, MPI_SUM, PCU_Get_Comm());
+
+ if (!PCU_Comm_Self())
+ {
+ int* global_entity_count = new int[4];
+ global_entity_count[0]=global_entity_count[1]=global_entity_count[2]=global_entity_count[3]=0;
+ for (int d=0; d<4;++d)
+ {
+ for (int p=0; p
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+double a_param = 0;
+double b_param = 0;
+double c_param = 0;
+double d_param = 0;
+double e_param = 0;
+double meshSize=0.1;
+void getCenter(double x[3])
+{
+ x[0]=a_param;
+ x[1]=d_param;
+ x[2]=0.;
+}
+void edgeFunction(double const p[2], double x[3], void*)
+{
+ double phi = p[0];
+ x[0] = a_param + b_param*(cos(phi + c_param*sin(phi)));
+ x[1] = d_param + e_param*sin(phi);
+ x[2] = 0;
+}
+
+void snap2Model(apf::Mesh2* mesh, gmi_model* model)
+{
+ for(int i=0; i<5; i++)
+ {
+ apf::MeshEntity* vt = getMdsEntity(mesh, 0, i);
+ assert(vt);
+ gmi_ent* gent= (gmi_ent*)(mesh->toModel(vt));
+ apf::Vector3 param(0,0,0);
+ int gType = gmi_dim(model,gent);
+ double xyz[3];
+ if(gType==1)
+ {
+ mesh->getParam(vt,param);
+ double pa[]={param[0],0,0};
+ edgeFunction(pa, xyz, 0);
+ }
+ else
+ {
+ assert(gType==2);
+ getCenter(xyz);
+ }
+ mesh->setPoint(vt, 0, xyz);
+ }
+}
+void faceFunction(double const p[2], double x[3], void*)
+{
+ (void)p;
+ (void)x;
+}
+
+gmi_model* makeModel(char* modelfile)
+{
+ FILE* fp=fopen(modelfile,"r");
+ if(!fp)
+ {
+ std::cout<<"SCOREC ERROR: fail to open the file \""<getDimension(); ++d) {
+ apf::MeshIterator* it = m->begin(d);
+ int i = 0;
+ apf::MeshEntity* e;
+ while ((e = m->iterate(it))) {
+ assert( apf::getMdsIndex(m, e) == i );
+ assert( apf::getMdsEntity(m, d, i) == e );
+ ++i;
+ }
+ m->end(it);
+ }
+}
+
+static void fusionAdapt(apf::Mesh2* m)
+{
+ Size sf(m);
+ for(int i=0; i<10; i++)
+ {
+ std::cout<<" iter "<maximumIterations = 1;
+ in->shouldRunPreZoltan = true;
+ in->shouldRunMidParma = true;
+ in->shouldRunPostParma = true;
+ ma::adapt(in);
+ }
+}
+
+int main( int argc, char* argv[])
+{
+ MPI_Init(&argc,&argv);
+ PCU_Comm_Init();
+
+ if (argc!=3)
+ {
+ std::cout<<"SCOREC ERROR: wrong input arguments. \nUsage: ./create_smb model_file relative_mesh_size"
+ <<"\n - model_file shall contain 5 double: a_param, b_param, c_param, d_param, e_param \n"
+ <<" - given relative_mesh_size, mesh edge length is about relative_mesh_size * the longest edge of the bounding box of the model\n";
+ PCU_Comm_Free();
+ MPI_Finalize();
+ return 1;
+ }
+
+ char* modelfile = argv[1];
+ meshSize =atof(argv[2]);
+
+ // create model
+ gmi_model* model = makeModel(modelfile);
+ if (!model)
+ {
+ PCU_Comm_Free();
+ MPI_Finalize();
+ return 1;
+ }
+ // generate and refine mesh
+ apf::Mesh2* mesh = apf::loadMdsMesh(model, "seed.smb");
+ snap2Model(mesh, model);
+ mesh->verify();
+ testIndexing(mesh);
+ fusionAdapt(mesh);
+ // export to files
+ char meshfile[256];
+ sprintf(meshfile,"%s.smb",modelfile);
+ mesh->writeNative(meshfile);
+ apf::writeVtkFiles("Vtk-Mesh-File", mesh);
+ std::cout<<"\n";
+ mesh->verify();
+ std::cout<<"\n";
+ // clean and finalize
+ mesh->destroyNative();
+ apf::destroyMesh(mesh);
+ PCU_Comm_Free();
+ MPI_Finalize();
+ return 0;
+}
diff --git a/m3dc1_scorec/utility/show_meshcount.cc b/m3dc1_scorec/utility/show_meshcount.cc
new file mode 100644
index 000000000..51a51b2b2
--- /dev/null
+++ b/m3dc1_scorec/utility/show_meshcount.cc
@@ -0,0 +1,78 @@
+/******************************************************************************
+
+ (c) 2005-2016 Scientific Computation Research Center,
+ Rensselaer Polytechnic Institute. All rights reserved.
+
+ This work is open source software, licensed under the terms of the
+ BSD license as described in the LICENSE file in the top-level directory.
+
+*******************************************************************************/
+#include
+#include
+#include
+#include "m3dc1_scorec.h"
+#include "m3dc1_mesh.h"
+#include "apfMesh.h"
+
+char* meshFile = 0;
+char* modelFile = 0;
+int num_plane=1;
+
+void getConfig(int argc, char** argv)
+{
+ if ( argc < 4 )
+ {
+ if ( !PCU_Comm_Self() )
+ printf("Usage: %s <#plane>\n", argv[0]);
+ m3dc1_scorec_finalize();
+ MPI_Finalize();
+ return;
+ }
+ modelFile = argv[1];
+ meshFile = argv[2];
+ if (atoi(argv[3])>1) num_plane=atoi(argv[3]);
+}
+
+int main(int argc, char** argv)
+{
+ MPI_Init(&argc,&argv);
+ m3dc1_scorec_init();
+
+ getConfig(argc,argv);
+
+ if (num_plane>1) // construct 3D
+ {
+ if (PCU_Comm_Peers()%num_plane!=0)
+ {
+ if ( !PCU_Comm_Self() )
+ printf("ERROR: invalid #planes\n");
+ m3dc1_scorec_finalize();
+ MPI_Finalize();
+ return 1;
+ }
+
+ m3dc1_model_setnumplane (&num_plane);
+ }
+
+
+ m3dc1_model_load(modelFile);
+ m3dc1_mesh_load(meshFile);
+// m3dc1_mesh::instance()->print(0);
+
+ if (num_plane>1)
+ {
+ int zero=0;
+ m3dc1_mesh_build3d(&zero, &zero, &zero);
+ }
+
+ // printStats(m3dc1_mesh::instance()->mesh);
+
+//Is there any way to tell how many vertices per plane after partition is applied from just the mesh file without going to actually M3DC1 code? And how many vertices per partition as well, without counting the ghost vertices which belong to other rank?
+ m3dc1_mesh::instance()->print(1);
+
+ // finalize
+ m3dc1_scorec_finalize();
+ MPI_Finalize();
+ return 0;
+}
+
diff --git a/m3dc1_scorec/utility/split_smb.cc b/m3dc1_scorec/utility/split_smb.cc
new file mode 100644
index 000000000..41275f108
--- /dev/null
+++ b/m3dc1_scorec/utility/split_smb.cc
@@ -0,0 +1,124 @@
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#ifdef HAVE_SIMMETRIX
+#include
+#include
+#endif
+#include
+#include
+#include
+#include
+
+namespace {
+
+const char* modelFile = 0;
+const char* meshFile = 0;
+const char* outFile = 0;
+int partitionFactor = 1;
+
+void freeMesh(apf::Mesh* m)
+{
+ m->destroyNative();
+ apf::destroyMesh(m);
+}
+
+apf::Migration* getPlan(apf::Mesh* m)
+{
+ apf::Splitter* splitter = apf::makeZoltanSplitter(
+ m, apf::GRAPH, apf::PARTITION, false);
+ apf::MeshTag* weights = Parma_WeighByMemory(m);
+ apf::Migration* plan = splitter->split(weights, 1.05, partitionFactor);
+ apf::removeTagFromDimension(m, weights, m->getDimension());
+ m->destroyTag(weights);
+ delete splitter;
+ return plan;
+}
+
+void switchToOriginals()
+{
+ int self = PCU_Comm_Self();
+ int groupRank = self / partitionFactor;
+ int group = self % partitionFactor;
+ MPI_Comm groupComm;
+ MPI_Comm_split(MPI_COMM_WORLD, group, groupRank, &groupComm);
+ PCU_Switch_Comm(groupComm);
+}
+
+void switchToAll()
+{
+ MPI_Comm prevComm = PCU_Get_Comm();
+ PCU_Switch_Comm(MPI_COMM_WORLD);
+ MPI_Comm_free(&prevComm);
+ PCU_Barrier();
+}
+
+void getConfig(int argc, char** argv)
+{
+ if ( argc < 4 ) {
+ if ( !PCU_Comm_Self() )
+ printf("Usage: %s \n"
+ "Increase the part count of mesh from inPartCount to #rank (inPartCount*factor).\n",
+ argv[0]);
+ MPI_Finalize();
+ exit(EXIT_FAILURE);
+ }
+ if (argc == 5)
+ {
+ meshFile = argv[2];
+ outFile = argv[3];
+ partitionFactor = atoi(argv[4]);
+ }
+ if (argc == 4)
+ {
+ meshFile = argv[1];
+ outFile = argv[2];
+ partitionFactor = atoi(argv[3]);
+ }
+ assert(partitionFactor <= PCU_Comm_Peers());
+}
+
+}
+
+int main(int argc, char** argv)
+{
+ MPI_Init(&argc,&argv);
+ PCU_Comm_Init();
+ double t0 = PCU_Time();
+#ifdef HAVE_SIMMETRIX
+ SimUtil_start();
+ Sim_readLicenseFile(0);
+ gmi_sim_start();
+ gmi_register_sim();
+#endif
+ gmi_register_null();
+ getConfig(argc,argv);
+ bool isOriginal = ((PCU_Comm_Self() % partitionFactor) == 0);
+ gmi_model* g = gmi_load(".null");
+ apf::Mesh2* m = 0;
+ apf::Migration* plan = 0;
+ switchToOriginals();
+ if (isOriginal) {
+ m = apf::loadMdsMesh(g, meshFile);
+ plan = getPlan(m);
+ }
+ switchToAll();
+ m = apf::repeatMdsMesh(m, g, plan, partitionFactor);
+ if (!PCU_Comm_Self())
+ std::cout<<"[PUMI INFO] mesh splitted into "<writeNative(outFile);
+ freeMesh(m);
+#ifdef HAVE_SIMMETRIX
+ gmi_sim_stop();
+ Sim_unregisterAllKeys();
+ SimUtil_stop();
+#endif
+ PCU_Comm_Free();
+ MPI_Finalize();
+}
diff --git a/m3dc1_scorec/utility/vtk_order.cc b/m3dc1_scorec/utility/vtk_order.cc
new file mode 100644
index 000000000..3e2a33e8c
--- /dev/null
+++ b/m3dc1_scorec/utility/vtk_order.cc
@@ -0,0 +1,180 @@
+/******************************************************************************
+
+ (c) 2005-2016 Scientific Computation Research Center,
+ Rensselaer Polytechnic Institute. All rights reserved.
+
+ This work is open source software, licensed under the terms of the
+ BSD license as described in the LICENSE file in the top-level directory.
+
+*******************************************************************************/
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+// seol -- reorder input mesh before conversion
+// starting vtx: a vtx with min Y
+struct Queue {
+ bool has(apf::MeshEntity* e) { return h.count(e); }
+ void push(apf::MeshEntity* e)
+ {
+ q.push_back(e);
+ h.insert(e);
+ }
+ void pushVector(std::vector const& l)
+ {
+ for (size_t i = 0; i < l.size(); ++i)
+ push(l[i]);
+ }
+ apf::MeshEntity* pop()
+ {
+ apf::MeshEntity* e;
+ e = q.front();
+ q.pop_front();
+ h.erase(e);
+ return e;
+ }
+ bool empty() { return q.empty(); }
+ std::deque q;
+ std::set h;
+};
+
+
+apf::MeshEntity* findFirst(apf::Mesh* m)
+{
+ apf::MeshEntity* v;
+ apf::MeshEntity* best;
+ apf::MeshIterator* it = m->begin(0);
+ best = m->iterate(it);
+ apf::Vector3 coord;
+ m->getPoint(best, 0, coord);
+ double min_Y=coord[1];
+ while ((v = m->iterate(it)))
+ {
+ m->getPoint(v, 0, coord);
+ if (min_Y > coord[1])
+ {
+ best = v;
+ min_Y = coord[1];
+ }
+ }
+ m->end(it);
+ return best;
+}
+
+bool visited(Queue& q, apf::Numbering* nn, apf::MeshEntity* e)
+{
+ return apf::isNumbered(nn, e, 0, 0) || q.has(e);
+}
+
+bool hasNode(apf::Mesh* m, apf::MeshEntity* e)
+{
+ if (m->getShape()->countNodesOn(m->getType(e))>0)
+ return true;
+ return false;
+}
+
+int main( int argc, char* argv[])
+{
+ MPI_Init(&argc,&argv);
+ PCU_Comm_Init();
+
+ if (argc!=2)
+ {
+ std::cout<<"SCOREC ERROR: wrong input arguments. \n"
+ <<" Usage: ./reorder inmesh.smb\n";
+ PCU_Comm_Free();
+ MPI_Finalize();
+ return 1;
+ }
+ gmi_register_null();
+ gmi_model* nullModel = gmi_load(".null");
+
+ apf::Mesh2* m = apf::loadMdsMesh(nullModel,argv[1]);
+ int mesh_dim = m->getDimension();
+
+ apf::Numbering* n_org=apf::numberOwnedDimension(m, "elem", mesh_dim);
+ writeVtkFiles("org-mesh.vtk", m);
+ std::cout << "[INFO] vtk files for original ordering saved in \"org-mesh.vtk\"\n";
+ destroyNumbering(n_org);
+
+ // reorder mesh
+ apf::Numbering* nn = apf::createNumbering(m, "node", apf::getConstant(0), 1);
+ apf::Numbering* en = apf::createNumbering(m, "elem", apf::getConstant(mesh_dim), 1);
+
+ Queue q;
+ q.push(findFirst(m));
+
+ // node and element number starts from 0
+ int labelnode = 0;
+ int labelelem = 0;
+
+ std::vector node_arr;
+ std::vector elem_arr;
+
+ node_arr.resize(m->count(0)+1);
+ elem_arr.resize(m->count(mesh_dim)+1);
+
+ apf::MeshEntity* otherVtx;
+ apf::MeshEntity* edge;
+ apf::MeshEntity* elem;
+
+ while (!q.empty())
+ {
+ apf::MeshEntity* vtx = q.pop();
+ if (!apf::isNumbered(nn, vtx, 0, 0))
+ {
+ node_arr[labelnode] = vtx;
+ apf::number(nn, vtx, 0, 0, labelnode);
+
+ ++labelnode;
+ }
+
+ std::vector entities;
+ apf::Adjacent edges;
+ m->getAdjacent(vtx,1, edges);
+ for (size_t i = 0; i < edges.getSize(); ++i)
+ {
+ edge = edges[i];
+ apf::Adjacent adjacent;
+ m->getAdjacent(edge, mesh_dim, adjacent);
+ for (size_t j = 0; j < adjacent.getSize(); ++j)
+ {
+ elem = adjacent[j];
+ if (!apf::isNumbered(en, elem, 0, 0))
+ {
+ elem_arr[labelelem] = elem;
+ apf::number(en, elem, 0, 0, labelelem);
+ ++labelelem;
+ }
+ }
+ otherVtx = apf::getEdgeVertOppositeVert(m, edge, vtx);
+ if (!visited(q, nn, otherVtx))
+ entities.push_back(otherVtx);
+ }
+ q.pushVector(entities);
+ } // while
+ destroyNumbering(nn);
+
+ writeVtkFiles("reordered-mesh.vtk", m);
+ std::cout << "[INFO] vtk files for adjacency-based ordering saved in \"reordered-mesh.vtk\"\n";
+ destroyNumbering(en);
+
+ m->destroyNative();
+ apf::destroyMesh(m);
+
+ PCU_Comm_Free();
+ MPI_Finalize();
+ return 0;
+}
diff --git a/skeleton/m3dc1_skeleton_big_spec_acc/main.F90 b/skeleton/m3dc1_skeleton_big_spec_acc/main.F90
index 257f47a46..6532fd3f8 100644
--- a/skeleton/m3dc1_skeleton_big_spec_acc/main.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_acc/main.F90
@@ -1,4 +1,5 @@
Program m3dc1_skeleton
+ use mpi
use mesh
use m3dc1_data
use matdef
@@ -7,7 +8,6 @@ Program m3dc1_skeleton
implicit none
- include 'mpif.h'
integer :: ierr
real :: tstart, tend
@@ -151,12 +151,12 @@ end subroutine rotation
! populates the *tri arrays
!============================================================
subroutine tridef
+ use mpi
use math
use mesh
implicit none
- include 'mpif.h'
type(element_data) :: d
integer :: itri, i, j, k, ii, jj, numelms, numnodes, ndofs
diff --git a/skeleton/m3dc1_skeleton_big_spec_acc/main_acc.F90 b/skeleton/m3dc1_skeleton_big_spec_acc/main_acc.F90
index 7a4ddfc2e..aba4be579 100644
--- a/skeleton/m3dc1_skeleton_big_spec_acc/main_acc.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_acc/main_acc.F90
@@ -1,4 +1,5 @@
Program m3dc1_skeleton
+ use mpi
use mesh
use m3dc1_data
use matdef
@@ -8,7 +9,6 @@ Program m3dc1_skeleton
implicit none
- include 'mpif.h'
integer :: ierr
real :: tstart, tend
@@ -176,12 +176,12 @@ end subroutine rotation
! populates the *tri arrays
!============================================================
subroutine tridef
+ use mpi
use math
use mesh
implicit none
- include 'mpif.h'
type(element_data) :: d
integer :: itri, i, j, k, ii, jj, numelms, numnodes, ndofs
diff --git a/skeleton/m3dc1_skeleton_big_spec_acc/main_omp.F90 b/skeleton/m3dc1_skeleton_big_spec_acc/main_omp.F90
index e796e8681..661b6b4ec 100644
--- a/skeleton/m3dc1_skeleton_big_spec_acc/main_omp.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_acc/main_omp.F90
@@ -1,4 +1,5 @@
Program m3dc1_skeleton
+ use mpi
use mesh
use m3dc1_data
use matdef
@@ -7,7 +8,6 @@ Program m3dc1_skeleton
implicit none
- include 'mpif.h'
integer :: ierr
real :: tstart, tend
@@ -151,12 +151,12 @@ end subroutine rotation
! populates the *tri arrays
!============================================================
subroutine tridef
+ use mpi
use math
use mesh
implicit none
- include 'mpif.h'
type(element_data) :: d
integer :: itri, i, j, k, ii, jj, numelms, numnodes, ndofs
diff --git a/skeleton/m3dc1_skeleton_big_spec_acc/mesh.F90 b/skeleton/m3dc1_skeleton_big_spec_acc/mesh.F90
index 0411d57a8..a4daa975c 100644
--- a/skeleton/m3dc1_skeleton_big_spec_acc/mesh.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_acc/mesh.F90
@@ -127,11 +127,11 @@ subroutine create_rectangular_mesh(m,n, nplanes,inodes, ielms, nodes, elms)
end subroutine create_rectangular_mesh
subroutine load_mesh(isize)
+ use mpi
use math
implicit none
- include "mpif.h"
integer :: isize, new_mesh_size
integer :: num_global_elements
diff --git a/skeleton/m3dc1_skeleton_big_spec_acc/mesh_acc.F90 b/skeleton/m3dc1_skeleton_big_spec_acc/mesh_acc.F90
index 03e4ff1e8..75d5b10b2 100644
--- a/skeleton/m3dc1_skeleton_big_spec_acc/mesh_acc.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_acc/mesh_acc.F90
@@ -132,11 +132,11 @@ subroutine create_rectangular_mesh(m,n, nplanes,inodes, ielms, nodes, elms)
end subroutine create_rectangular_mesh
subroutine load_mesh(isize)
+ use mpi
use math
implicit none
- include "mpif.h"
integer :: isize, new_mesh_size
integer :: num_global_elements
diff --git a/skeleton/m3dc1_skeleton_big_spec_omp/main.F90 b/skeleton/m3dc1_skeleton_big_spec_omp/main.F90
index d49e9787d..133a0eb5e 100644
--- a/skeleton/m3dc1_skeleton_big_spec_omp/main.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_omp/main.F90
@@ -1,4 +1,5 @@
Program m3dc1_skeleton
+ use mpi
use mesh
use m3dc1_data
use matdef
@@ -7,7 +8,6 @@ Program m3dc1_skeleton
implicit none
- include 'mpif.h'
integer :: ierr
real :: tstart, tend
@@ -146,12 +146,12 @@ end subroutine rotation
! populates the *tri arrays
!============================================================
subroutine tridef
+ use mpi
use math
use mesh
implicit none
- include 'mpif.h'
type(element_data) :: d
integer :: itri, i, j, k, ii, jj, numelms, numnodes, ndofs
diff --git a/skeleton/m3dc1_skeleton_big_spec_omp/mesh.F90 b/skeleton/m3dc1_skeleton_big_spec_omp/mesh.F90
index c2f8aa99f..849ea2f9a 100644
--- a/skeleton/m3dc1_skeleton_big_spec_omp/mesh.F90
+++ b/skeleton/m3dc1_skeleton_big_spec_omp/mesh.F90
@@ -127,11 +127,11 @@ subroutine create_rectangular_mesh(m,n, nplanes,inodes, ielms, nodes, elms)
end subroutine create_rectangular_mesh
subroutine load_mesh
+ use mpi
use math
implicit none
- include "mpif.h"
integer :: num_global_elements
type(node_type), allocatable :: global_node(:)
diff --git a/unstructured/CMakeLists.txt b/unstructured/CMakeLists.txt
index ac86dd2ad..28ee00258 100644
--- a/unstructured/CMakeLists.txt
+++ b/unstructured/CMakeLists.txt
@@ -1,487 +1,379 @@
-cmake_minimum_required(VERSION 3.12)
-
-# set(CMAKE_DISABLE_SOURCE_CHANGES ON)
-# set(CMAKE_DISABLE_IN_SOURCE_BUILD ON)
-
-project(m3dc1_unstructured LANGUAGES Fortran C CXX)
-
-
-if("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")
- message(SEND_ERROR "In-source builds are not allowed.")
-endif("${PROJECT_SOURCE_DIR}" STREQUAL "${PROJECT_BINARY_DIR}")
-
-# enable_language(Fortran)
-
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-
-if(NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Release)
+set(M3DC1_MESH_MOD "scorec_mesh_mod" CACHE STRING
+ "Mesh module name")
+set(M3DC1_VECTOR_MOD "scorec_vector_mod" CACHE STRING
+ "Vector module name")
+set(M3DC1_MATRIX_MOD "scorec_matrix_mod" CACHE STRING
+ "Matrix module name")
+set(M3DC1_VECTOR_TYPE "scorec_vector" CACHE STRING
+ "Vector derived-type name")
+set(M3DC1_MATRIX_TYPE "scorec_matrix" CACHE STRING
+ "Matrix derived-type name")
+
+message(STATUS "M3DC1 mesh backend : ${M3DC1_MESH_MOD}")
+message(STATUS "M3DC1 vector backend : ${M3DC1_VECTOR_MOD} (type = ${M3DC1_VECTOR_TYPE})")
+message(STATUS "M3DC1 matrix backend : ${M3DC1_MATRIX_MOD} (type = ${M3DC1_MATRIX_TYPE})")
+
+
+set(M3DC1_LIBRARY_SOURCES
+ adapt.f90
+ auxiliary_fields.f90
+ biharmonic.f90
+ bootstrap.f90
+ boundary.f90
+ coil_sets.f90
+ coils.f90
+ control.f90
+ diagnostics.f90
+ electric_field.f90
+ electrostatic_potential.f90
+ element.f90
+ error_estimate.f90
+ field.f90
+ fit_magnetics.f90
+ fftw_fortran.c
+ geometry.f90
+ get_pc_skip_count.f90
+ gsl_wrapper.c
+ gradshafranov.f90
+ gyroviscosity.f90
+ harned_mikic.f90
+ hdf5_output.f90
+ hypervisc.f90
+ init_3ddiffusion.f90
+ init_3dwave.f90
+ init_basicj.f90
+ init_basicq.f90
+ init_circle.f90
+ init_common.f90
+ init_conds.f90
+ init_cyl.f90
+ init_dskbal.f90
+ init_eigen.f90
+ init_eqdsk.f90
+ init_force_free.f90
+ init_frs.f90
+ init_ftz.f90
+ init_gem.f90
+ init_gmode.f90
+ init_intkink.f90
+ init_jsolver.f90
+ read_jsolver.f90
+ readaeqdsk.f90
+ init_kstar.f90
+ init_lz.f90
+ init_mri.f90
+ init_rotating_cylinder.f90
+ init_rwm.f90
+ init_solovev.f90
+ init_strauss.f90
+ init_taylor.f90
+ init_tilt.f90
+ init_vmec.f90
+ init_wave.f90
+ input.f90
+ interpolate.f90
+ iterdb.f90
+ kprad_m3dc1.f90
+ kprad.f90
+ ludef_t.f90
+ m3dc1_nint.f90
+ M3Dmodules.f90
+ math.f90
+ metricterms_new.f90
+ model.f90
+ neutral_beam.f90
+ newvar.f90
+ newpar.f90
+ nintegrate_mod.f90
+ output.f90
+ parallel_heat_flux.f90
+ parse_solver_options.c
+ particle.f90
+ pellet.f90
+ physical_mesh.f90
+ polar.f90
+ radiation.f90
+ random.c
+ read_ascii.f90
+ read_dskbal.f90
+ read_gyro.f90
+ read_neo.f90
+ read_namelist.cpp
+ read_schaffer_field.f90
+ read_vmec.f90
+ readgeqdsk.f90
+ region.f90
+ resistive_wall.f90
+ restart_hdf5.f90
+ rmp.f90
+ runaway_advection.f90
+ runaway.f90
+ scorec_adapt.f90
+ signal_handler.f90
+ spline.f90
+ temperature_plots.f90
+ time_step_split.f90
+ time_step_unsplit.f90
+ time_step.f90
+ transport.f90
+ two_fluid.f90
+ wall.f90
+ mesh_mod.F90
+ vector_mod.F90
+ matrix_mod.F90
+)
+
+if(M3DC1_MESH_MOD STREQUAL "scorec_mesh_mod")
+ list(APPEND M3DC1_LIBRARY_SOURCES scorec_mesh.f90)
+ set(M3DC1_MESH_DEFINE M3DC1_MESH_IS_SCOREC)
+elseif(M3DC1_MESH_MOD STREQUAL "basic_mesh_mod")
+ list(APPEND M3DC1_LIBRARY_SOURCES basic_mesh.f90)
+ set(M3DC1_MESH_DEFINE M3DC1_MESH_IS_BASIC)
+else()
+ message(FATAL_ERROR "Unknown M3DC1_MESH_MOD=${M3DC1_MESH_MOD}")
endif()
-# set(CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".so")
-# if(BUILD_SHARED_LIBS)
- # set(CMAKE_FIND_LIBRARY_SUFFIXES ".so" ".a")
-# endif()
-message(STATUS "Found PETSc ${CMAKE_FIND_LIBRARY_SUFFIXES}")
-
-# set(CMAKE_CXX_FLAGS "-Wall -Wextra")
-# set(CMAKE_CXX_FLAGS_DEBUG "-g")
-# set(CMAKE_CXX_FLAGS_RELEASE "-O3")
-
-set(CMAKE_Fortran_PREPROCESS "ON")
-
-# option(BUILD_SHARED_LIBS "Build using shared libraries" OFF)
-
-set(SOURCES
- scorec_mesh.f90
- scorec_vector.f90
- scorec_matrix.f90
- scorec_adapt.f90
- PETScInterface.cpp
- physical_mesh.f90
- read_vmec.f90
- fftw_fortran.c
- read_namelist.cpp
- gsl_wrapper.c
- # random.c
- spline.f90
- kprad.f90
- math.f90
- read_ascii.f90
- interpolate.f90
- control.f90
- iterdb.f90
- read_gyro.f90
- read_neo.f90
- region.f90
- radiation.f90
- element.f90
- field.f90
- nintegrate_mod.f90
- M3Dmodules.f90
- resistive_wall.f90
- m3dc1_nint.f90
- boundary.f90
- gyroviscosity.f90
- bootstrap.f90
- metricterms_new.f90
- # metricterms_new_gpu.f90
- two_fluid.f90
- harned_mikic.f90
- biharmonic.f90
- electric_field.f90
- pellet.f90
- parallel_heat_flux.f90
- temperature_plots.f90
- electrostatic_potential.f90
- newvar.f90
- runaway.f90
- diagnostics.f90
- read_schaffer_field.f90
- neutral_beam.f90
- coils.f90
- coil_sets.f90
- model.f90
- fit_magnetics.f90
- wall.f90
- init_common.f90
- gradshafranov.f90
- rmp.f90
- kprad_m3dc1.f90
- readgeqdsk.f90
- read_jsolver.f90
- read_dskbal.f90
- init_cyl.f90
- init_rwm.f90
- init_solovev.f90
- init_circle.f90
- init_basicj.f90
- init_tilt.f90
- init_taylor.f90
- init_force_free.f90
- init_gem.f90
- init_wave.f90
- init_gmode.f90
- init_strauss.f90
- init_mri.f90
- init_rotating_cylinder.f90
- init_eqdsk.f90
- init_dskbal.f90
- init_jsolver.f90
- init_3dwave.f90
- init_3ddiffusion.f90
- init_frs.f90
- init_ftz.f90
- init_eigen.f90
- init_intkink.f90
- init_lz.f90
- init_kstar.f90
- init_basicq.f90
- init_vmec.f90
- transport.f90
- auxiliary_fields.f90
- runaway_advection.f90
- time_step_split.f90
- time_step_unsplit.f90
- hdf5_output.f90
- particle.f90
- time_step.f90
- output.f90
- restart_hdf5.f90
- error_estimate.f90
- adapt.f90
- input.f90
- ludef_t.f90
- # ludef_t_gpu.f90
- init_conds.f90
- # get_pc_skip_count.f90
- hypervisc.f90
- geometry.f90
- newpar.f90
- )
-
-set(BIN_NAME m3dc1)
-
-# find_package(PETSc)
-# if(PETSC_FOUND AND USEPETSC)
-# set(PETSC_SOURCES
-# basic_mesh.f90
-# petsc_vector.f90
-# petsc_matrix.f90)
-# set(SOURCES "${SOURCES};${PETSC_SOURCES}")
-# list(APPEND DEFS -DUSEPETSC -Dmesh_mod=basic_mesh_mod -Dvector_mod=petsc_vector_mod -Dmatrix_mod=petsc_matrix_mod)
-# set(POSTFIX "${POSTFIX}-petsc")
-# elseif(PETSC_FOUND)
-# include(${CMAKE_SOURCE_DIR}/cmake/petsc.cmake)
-# petsc_get_version()
-# message(STATUS "Petsc version detected as : ${PETSC_VERSION}")
-# list(APPEND DEFS -DPETSC_FORTRAN_PETSCTRUTH_INT -DPETSC_VERSION=${PETSC_VERSION})
-# endif()
-find_package(PkgConfig REQUIRED)
-# list(APPEND PKG_CONFIG_EXECUTABLE "--static")
-
-# pkg_check_modules (PETSC REQUIRED PETSc)
-pkg_search_module(PETSC PETSc IMPORTED_TARGET REQUIRED)
-message(STATUS "Found PETSc ${PETSC_VERSION}")
-message(STATUS "Found PETSc ${CMAKE_Fortran_COMPILER_ID}")
-
-# foreach(ITR ${PETSC_LINK_LIBRARIES})
-# if(ITR MATCHES "liblapack\.a")
-# set(LAPACK_LIBRARY ${ITR})
-# endif()
-# endforeach()
-# list(FILTER PETSC_LINK_LIBRARIES EXCLUDE REGEX "liblapack\.a")
-# foreach(ITR ${PETSC_LINK_LIBRARIES})
-# if(ITR MATCHES "libblas\.a")
-# list(FIND PETSC_LINK_LIBRARIES ${ITR} IDX)
-# endif()
-# endforeach()
-# list(INSERT PETSC_LINK_LIBRARIES ${IDX} ${LAPACK_LIBRARY})
-# message(STATUS "Found PETSc ${PETSC_STATIC_LIBRARIES}")
-set(PETSC_LINK_LIBRARIES "")
- # message(STATUS "Found PETSc ${PETSC_LINK_LIBRARIES}")
-message(STATUS "Found PETSc ${PETSC_STATIC_LDFLAGS}")
-# list(FIND PETSC_STATIC_LDFLAGS "-lmetis" _index)
-# math(EXPR _index "${_index}+1")
-# message(STATUS "Found PETSc ${_index}")
-# list(SUBLIST PETSC_STATIC_LDFLAGS 0 ${_index} PETSC_STATIC_LDFLAGS_MODIFIED)
-# message(STATUS "Found PETSc ${PETSC_STATIC_LDFLAGS_MODIFIED}")
+if(M3DC1_VECTOR_MOD STREQUAL "scorec_vector_mod")
+ list(APPEND M3DC1_LIBRARY_SOURCES scorec_vector.f90)
+ set(M3DC1_VECTOR_DEFINE M3DC1_VECTOR_IS_SCOREC)
+elseif(M3DC1_VECTOR_MOD STREQUAL "petsc_vector_mod")
+ list(APPEND M3DC1_LIBRARY_SOURCES petsc_vector.f90)
+ set(M3DC1_VECTOR_DEFINE M3DC1_VECTOR_IS_PETSC)
+else()
+ message(FATAL_ERROR "Unknown M3DC1_VECTOR_MOD=${M3DC1_VECTOR_MOD}")
+endif()
-# if(ENABLE_STATIC)
-# set(STATIC_LIST lapack blas)
-# endif()
-foreach (flag IN LISTS PETSC_STATIC_LDFLAGS)
- if (flag MATCHES "^-L(.*)")
- list(APPEND _search_paths ${CMAKE_MATCH_1})
- continue()
- endif()
- if (flag MATCHES "^-l(.*)")
- set(_pkg_search "${CMAKE_MATCH_1}")
- endif()
+if(M3DC1_MATRIX_MOD STREQUAL "scorec_matrix_mod")
+ list(APPEND M3DC1_LIBRARY_SOURCES scorec_matrix.f90)
+ set(M3DC1_MATRIX_DEFINE M3DC1_MATRIX_IS_SCOREC)
+elseif(M3DC1_MATRIX_MOD STREQUAL "petsc_matrix_mod")
+ list(APPEND M3DC1_LIBRARY_SOURCES petsc_matrix.f90)
+ set(M3DC1_MATRIX_DEFINE M3DC1_MATRIX_IS_PETSC)
+else()
+ message(FATAL_ERROR "Unknown M3DC1_MATRIX_MOD=${M3DC1_MATRIX_MOD}")
+endif()
- # if (${_pkg_search} IN_LIST STATIC_LIST)
- # message(STATUS "Found PETSc ${_pkg_search}")
- # set(_pkg_search "lib${_pkg_search}.a")
- # endif()
+if(M3DC1_ENABLE_GPU)
+ list(REMOVE_ITEM M3DC1_SOURCES ludef_t.f90 metricterms_new.f90)
+ list(APPEND M3DC1_SOURCES ludef_t_gpu.f90 metricterms_new_gpu.f90)
+endif()
- if(_search_paths)
- # Firstly search in -L paths
- find_library(pkgcfg_lib_PETSC_${_pkg_search}
- NAMES ${_pkg_search}
- HINTS ${_search_paths} NO_DEFAULT_PATH)
- endif()
- find_library(pkgcfg_lib_PETSC_${_pkg_search}
- NAMES ${_pkg_search})
- if(pkgcfg_lib_PETSC_${_pkg_search})
- list(APPEND PETSC_LINK_LIBRARIES "${pkgcfg_lib_PETSC_${_pkg_search}}")
- else()
- list(APPEND PETSC_LINK_LIBRARIES ${_pkg_search})
- endif()
+if(NOT DEFINED M3DC1_ADAS_DATA_DIR)
+ set(M3DC1_ADAS_DATA_DIR "${CMAKE_BINARY_DIR}/adas/adf11"
+ CACHE PATH "Path to OpenADAS ADF11 data directory")
+endif()
+set(M3DC1_OPENADAS_SOURCE_DIR "${CMAKE_BINARY_DIR}/adas/source")
+set(M3DC1_OPENADAS_SOURCE_NAMES
+ xxrptn
+ xfelem
+ i4unit
+ i4fctn
+ xxword
+ xxcase
+ xxslen
+ xxdata_11
+)
+foreach(_openadas_source_name IN LISTS M3DC1_OPENADAS_SOURCE_NAMES)
+ list(APPEND M3DC1_OPENADAS_SOURCES
+ "${M3DC1_OPENADAS_SOURCE_DIR}/${_openadas_source_name}.f")
endforeach()
-# message(STATUS "Found PETSc ${PETSC_LIBRARIES}")
-message(STATUS "Found PETSc ${PETSC_LINK_LIBRARIES}")
-# list(TRANSFORM PETSC_LINK_LIBRARIES REPLACE "libpthread\.a" "libpthread.so")
-# list(FILTER PETSC_LINK_LIBRARIES EXCLUDE REGEX "libpthread")
-list(FILTER PETSC_LINK_LIBRARIES EXCLUDE REGEX "libcuda\.so")
-# list(FILTER PETSC_LINK_LIBRARIES EXCLUDE REGEX "libsci_cray\.a")
-# list(REMOVE_DUPLICATES PETSC_LINK_LIBRARIES)
-set_property(TARGET PkgConfig::PETSC PROPERTY
- INTERFACE_LINK_LIBRARIES "${PETSC_LINK_LIBRARIES}")
-
-pkg_search_module(FFTW fftw3 IMPORTED_TARGET REQUIRED)
-
-pkg_search_module(GSL gsl IMPORTED_TARGET REQUIRED)
-# list(FILTER GSL_LINK_LIBRARIES EXCLUDE REGEX "libm")
-# list(TRANSFORM GSL_LINK_LIBRARIES REPLACE "libm\.a" "libm.so")
-message(STATUS "Found gsl ${GSL_LINK_LIBRARIES}")
-# set_property(TARGET PkgConfig::GSL PROPERTY
-# INTERFACE_LINK_LIBRARIES "${GSL_LINK_LIBRARIES}")
-
-#find_package(TRILINOS)
-# if(TRILINOS_FOUND AND USETRILINOS)
-# list(APPEND DEFS -DM3DC1_TRILINOS)
-# set(POSTFIX "${POSTFIX}-trilinos")
-# endif()
-
-find_package(Scorec)
-# if(SCOREC_FOUND AND USESCOREC)
-# if(TRILINOS_FOUND AND USETRILINOS)
-# set(SCOREC_SOURCES
-# scorec_mesh.f90
-# scorec_vector.f90
-# scorec_matrix.f90)
-# endif()
-# if(PETSC_FOUND AND USESCOREC)
-# set(SCOREC_SOURCES
-# scorec_mesh.f90
-# scorec_vector.f90
-# scorec_matrix.f90
-# PETScInterface.cpp)
-# endif()
-# set(SOURCES "${SOURCES};${SCOREC_SOURCES}")
-# list(APPEND DEFS -DUSESCOREC -Dmesh_mod=scorec_mesh_mod -Dvector_mod=scorec_vector_mod -Dmatrix_mod=scorec_matrix_mod -Dmatrix_type=scorec_matrix -Dvector_type=scorec_vector)
-# if(USERW)
-# list(APPEND DEFS -DUSERW)
-# set(POSTFIX "${POSTFIX}-rw")
-# else()
-# list(APPEND DEFS -Dglobalinsertval=insertval -Dglobalentdofs=entdofs)
-# endif()
-# endif()
-
-# if(MATRIX_DUMP)
-# list(APPEND DEFS -DMATRIX_DUMP)
-# endif()
-
-find_package(Zoltan)
-#find_package(PARMETIS)
-#find_package(METIS)
-
-find_package(ZLIB)
-message(STATUS "ZLIB found: ${ZLIB_LIBRARIES}")
-find_package(HDF5 REQUIRED COMPONENTS Fortran HL)
-message(STATUS "HDF5 found: ${HDF5_HL_LIBRARIES}")
-if (HDF5_HL_LIBRARIES MATCHES "libhdf5_hl_fortran\.a")
- set(HDF5_USE_STATIC_LIBRARIES TRUE)
- find_package(HDF5 REQUIRED COMPONENTS Fortran HL)
- message(STATUS "HDF5 found: ${HDF5_HL_LIBRARIES}")
+set_source_files_properties(${M3DC1_OPENADAS_SOURCES} PROPERTIES GENERATED TRUE)
+set(M3DC1_OPENADAS_DATA_FILES
+ "${M3DC1_ADAS_DATA_DIR}/scd85/scd85_ar.dat"
+ "${M3DC1_ADAS_DATA_DIR}/scd89/scd89_b.dat"
+ "${M3DC1_ADAS_DATA_DIR}/plt89/plt89_b.dat"
+ "${M3DC1_ADAS_DATA_DIR}/plt89/plt89_ar.dat"
+ "${M3DC1_ADAS_DATA_DIR}/scd96/scd96_he.dat"
+ "${M3DC1_ADAS_DATA_DIR}/scd96/scd96_be.dat"
+ "${M3DC1_ADAS_DATA_DIR}/scd96/scd96_c.dat"
+ "${M3DC1_ADAS_DATA_DIR}/scd96/scd96_ne.dat"
+ "${M3DC1_ADAS_DATA_DIR}/plt96/plt96_he.dat"
+ "${M3DC1_ADAS_DATA_DIR}/plt96/plt96_be.dat"
+ "${M3DC1_ADAS_DATA_DIR}/plt96/plt96_c.dat"
+ "${M3DC1_ADAS_DATA_DIR}/plt96/plt96_ne.dat"
+)
+set(M3DC1_OPENADAS_STAMP "${CMAKE_BINARY_DIR}/adas/.openadas.stamp")
+add_custom_command(
+ OUTPUT
+ "${M3DC1_OPENADAS_STAMP}"
+ ${M3DC1_OPENADAS_SOURCES}
+ ${M3DC1_OPENADAS_DATA_FILES}
+ COMMAND "${CMAKE_COMMAND}"
+ "-DOUT_DIR=${CMAKE_BINARY_DIR}/adas"
+ "-DADF11_DIR=${M3DC1_ADAS_DATA_DIR}"
+ -P "${PROJECT_SOURCE_DIR}/cmake/DownloadOpenADAS.cmake"
+ COMMAND "${CMAKE_COMMAND}" -E touch "${M3DC1_OPENADAS_STAMP}"
+ COMMENT "Downloading OpenADAS ADF11 data and reader sources"
+ VERBATIM)
+set(_download_open_adas_all)
+if(M3DC1_DOWNLOAD_ADAS_DATA)
+ set(_download_open_adas_all ALL)
endif()
-if (HDF5_FOUND)
- message(STATUS "HDF5 found: ${HDF5_VERSION}")
-else ()
- message(FATAL_ERROR "HDF5 not found.")
-endif ()
-
-pkg_search_module(NETCDF netcdf IMPORTED_TARGET REQUIRED)
-message(STATUS "Found NetCDF ${NETCDF_VERSION}")
-pkg_search_module(NETCDF-Fortran netcdf-fortran IMPORTED_TARGET REQUIRED)
-message(STATUS "Found NetCDF-Fortran ${NETCDF-Fortran_VERSION}")
-
-# find_package(netCDF-Fortran REQUIRED)
-# if (netCDF_FOUND)
-# message(STATUS "netCDF found: ${netCDF_VERSION}")
-# else ()
-# message(FATAL_ERROR "netCDF not found.")
-# endif ()
+add_custom_target(download_open_adas ${_download_open_adas_all}
+ DEPENDS "${M3DC1_OPENADAS_STAMP}")
-#find_package(ZLIB)
-# find_package(GSL REQUIRED)
-# find_package(FFTW REQUIRED)
+if(M3DC1_ENABLE_ADAS)
+ list(APPEND M3DC1_LIBRARY_SOURCES adas_m3dc1.f90)
+ add_library(openadas_reader STATIC ${M3DC1_OPENADAS_SOURCES})
+ add_dependencies(openadas_reader download_open_adas)
+endif()
find_package(MPI REQUIRED)
-get_property(MPI_LINK_LIBRARIES TARGET MPI::MPI_Fortran PROPERTY INTERFACE_LINK_LIBRARIES)
-list(FILTER MPI_LINK_LIBRARIES EXCLUDE REGEX "libcuda\.so")
-set_property(TARGET MPI::MPI_Fortran PROPERTY
- INTERFACE_LINK_LIBRARIES "${MPI_LINK_LIBRARIES}")
+find_package(HDF5 COMPONENTS Fortran HL REQUIRED)
+find_package(GSL REQUIRED)
+find_package(LAPACK REQUIRED)
+find_path(FFTW3_INCLUDE_DIR NAMES fftw3.h REQUIRED)
+find_library(FFTW3_LIBRARY NAMES fftw3 REQUIRED)
+
+if(M3DC1_ENABLE_ST)
+ find_library(NETCDF_C_LIBRARY NAMES netcdf
+ HINTS ${NetCDF_ROOT}
+ PATH_SUFFIXES lib lib64
+ REQUIRED)
+endif()
-set(M3DC1_SCOREC_LIB_NAME m3dc1_scorec)
-if(ENABLE_3D)
- set(BIN_NAME "${BIN_NAME}_3d")
- list(APPEND DEFS -DUSE3D -Dvectype=real -DMAX_PTS=125)
- if(ENABLE_ST)
- set(BIN_NAME "${BIN_NAME}_st")
- list(APPEND DEFS -DUSEST)
- # if(PAR)
- # set(POSTFIX "${POSTFIX}-pix")
- # else()
- # set(POSTFIX "${POSTFIX}-complex")
- # endif()
- else()
- list(APPEND DEFS -Dvectype=real)
- endif()
- # set(POSTFIX "${POSTFIX}-125")
+if(PETSC_VERSION MATCHES "^([0-9]+)\\.([0-9]+)")
+ math(EXPR M3DC1_PETSC_VERSION_CODE "${CMAKE_MATCH_1} * 10 + ${CMAKE_MATCH_2}")
else()
- set(BIN_NAME "${BIN_NAME}_2d")
- list(APPEND DEFS -DMAX_PTS=25)
- # set(POSTFIX "${POSTFIX}-25")
- if(ENABLE_COMPLEX)
- set(BIN_NAME "${BIN_NAME}_complex")
- list(APPEND DEFS -Dvectype=complex -DUSECOMPLEX)
- set(M3DC1_SCOREC_LIB_NAME ${M3DC1_SCOREC_LIB_NAME}_complex)
- # if(PAR)
- # set(POSTFIX "${POSTFIX}-pix")
- # else()
- # set(POSTFIX "${POSTFIX}-complex")
- # endif()
- else()
- list(APPEND DEFS -Dvectype=real)
- endif()
+ set(M3DC1_PETSC_VERSION_CODE 0)
endif()
-if(ENABLE_PARTICLE)
- list(APPEND DEFS -DUSEPARTICLES)
-endif()
-
-# if(USEOMP)
-# set(POSTFIX "${POSTFIX}-omp")
-# endif()
-
-execute_process(COMMAND date OUTPUT_VARIABLE DATE)
-string(REGEX REPLACE "\n$" "" DATE "${DATE}")
-message(STATUS "Build date : ${DATE}")
-execute_process(COMMAND cat ${CMAKE_CURRENT_SOURCE_DIR}/release_version OUTPUT_VARIABLE VERSION)
-string(REGEX REPLACE "\n$" "" VERSION "${VERSION}")
-message(STATUS "Release version : ${VERSION}")
-list(APPEND DEFS -DUSESCOREC -Dmesh_mod=scorec_mesh_mod -Dvector_mod=scorec_vector_mod -Dmatrix_mod=scorec_matrix_mod -Dmatrix_type=scorec_matrix -Dvector_type=scorec_vector -DPETSC_VERSION=990 -DUSEBLAS -DDATE_BUILT='${DATE}' -DRELEASE_VERSION='${VERSION}')
-
-# if(NOT ARCH)
-# if(M3DC1_ARCH)
-# set(ARCH ${M3DC1_ARCH})
-# else()
-# set(ARCH ${HOST})
-# endif()
-# endif()
-# set(POSTFIX "${ARCH}${POSTFIX}")
-
-# include(cmake/${ARCH}.cmake)
-
-#set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -ffree-line-length-none")
-
-#todo : build depency list as we go instead of adding all at the end (trilinos, etc)
-
-# if(COM)
-# set(BIN_NAME "${BIN_NAME}_complex")
-# endif()
-
-set_source_files_properties(
- scorec_vector.f90
- PROPERTIES
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${BIN_NAME}.dir/scorec_mesh.f90.o
+string(TIMESTAMP M3DC1_DATE_BUILT "%a %b %d %H:%M:%S %Y")
+file(READ "${CMAKE_CURRENT_SOURCE_DIR}/release_version" M3DC1_RELEASE_VERSION)
+string(STRIP "${M3DC1_RELEASE_VERSION}" M3DC1_RELEASE_VERSION)
+
+add_library(m3dc1_library ${M3DC1_LIBRARY_SOURCES})
+
+target_include_directories(m3dc1_library PRIVATE ${FFTW3_INCLUDE_DIR})
+target_link_libraries(m3dc1_library PUBLIC
+ m3dc1::scorec
+ MPI::MPI_Fortran
+ PETSc::PETSc
+ hdf5::hdf5_fortran
+ hdf5::hdf5_hl_fortran
+ GSL::gsl
+ LAPACK::LAPACK
+ ${FFTW3_LIBRARY}
+)
+if(M3DC1_ENABLE_ST)
+ target_link_libraries(m3dc1_library PUBLIC
+ ${NETCDF_C_LIBRARY}
)
-
-set_source_files_properties(
- scorec_matrix.f90
- PROPERTIES
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${BIN_NAME}.dir/scorec_vector.f90.o
- )
-
-set_source_files_properties(
- field.f90
- PROPERTIES
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${BIN_NAME}.dir/scorec_vector.f90.o
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${BIN_NAME}.dir/scorec_matrix.f90.o
- )
-
-set_source_files_properties(
- M3Dmodules.f90
- PROPERTIES
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${BIN_NAME}.dir/scorec_mesh.f90.o
- )
-
-set_source_files_properties(
- nintegrate_mod.f90
- PROPERTIES
- OBJECT_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/${BIN_NAME}.dir/scorec_mesh.f90.o
- )
-
-if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
- set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
- set_source_files_properties( scorec_mesh.f90 PROPERTIES COMPILE_FLAGS -O0 )
- set_source_files_properties( transport.f90 PROPERTIES COMPILE_FLAGS -O0 )
- set_source_files_properties( particle.f90 PROPERTIES COMPILE_FLAGS -O1 )
- set_source_files_properties( ludef_t.f90 PROPERTIES COMPILE_FLAGS -O1 )
- set_source_files_properties( ludef_t_gpu.f90 PROPERTIES COMPILE_FLAGS -O1 )
endif()
-add_executable(${BIN_NAME} ${SOURCES})
-# set_target_properties(${BIN_NAME} PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${BIN_NAME}_mods)
-set_property(TARGET ${BIN_NAME} PROPERTY LINKER_LANGUAGE Fortran)
-target_include_directories(${BIN_NAME} PRIVATE ${SCOREC_INCLUDE_DIRS})
-target_link_libraries(${BIN_NAME} PRIVATE PkgConfig::NETCDF-Fortran PkgConfig::NETCDF
- ${HDF5_HL_LIBRARIES} ${HDF5_LIBRARIES}
- # netCDF::netcdff
- ${M3DC1_SCOREC_LIB_NAME} ${SCOREC_LIBRARIES} ${ZOLTAN_LIBRARIES}
- PkgConfig::PETSC PkgConfig::FFTW PkgConfig::GSL MPI::MPI_Fortran stdc++)
-# target_link_directories(m3dc1_2d ${PETSC_STATIC_LIBRARY_DIRECTORIES})
-target_compile_definitions(${BIN_NAME} PRIVATE ${DEFS})
-if (CMAKE_Fortran_COMPILER_ID STREQUAL "Cray")
- target_compile_options(${BIN_NAME} PRIVATE $<$:-s real64>)
-elseif (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU")
- target_compile_options(${BIN_NAME} PRIVATE $<$:-fdefault-real-8>)
- if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10)
- target_compile_options(${BIN_NAME} PRIVATE $<$:-fallow-argument-mismatch>)
- endif()
- target_compile_options(${BIN_NAME} PRIVATE $<$:-ffree-line-length-512>)
+if(M3DC1_ENABLE_ADAS)
+ target_link_libraries(m3dc1_library PRIVATE openadas_reader)
+endif()
+target_compile_definitions(m3dc1_library PRIVATE
+ USESCOREC
+ ${M3DC1_MESH_DEFINE}
+ ${M3DC1_VECTOR_DEFINE}
+ ${M3DC1_MATRIX_DEFINE}
+ vector_type=${M3DC1_VECTOR_TYPE}
+ matrix_type=${M3DC1_MATRIX_TYPE}
+ PETSC_VERSION=${M3DC1_PETSC_VERSION_CODE}
+ DATE_BUILT="${M3DC1_DATE_BUILT}"
+ RELEASE_VERSION="${M3DC1_RELEASE_VERSION}"
+)
+if(M3DC1_ENABLE_3D)
+ target_compile_definitions(m3dc1_library PRIVATE USE3D MAX_PTS=125)
else()
- target_compile_options(${BIN_NAME} PRIVATE $<$:-r8>)
+ target_compile_definitions(m3dc1_library PRIVATE MAX_PTS=25)
endif()
-install(TARGETS ${BIN_NAME} DESTINATION ".")
-# get_property(TTT TARGET m3dc1_3d PROPERTY
-# INTERFACE_LINK_LIBRARIES)
-list(FILTER CMAKE_C_IMPLICIT_LINK_LIBRARIES EXCLUDE REGEX "^c$")
-list(FILTER CMAKE_CXX_IMPLICIT_LINK_LIBRARIES EXCLUDE REGEX "^c$")
-message(STATUS ${CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES})
-message(STATUS ${CMAKE_C_IMPLICIT_LINK_LIBRARIES})
-message(STATUS ${CMAKE_CXX_IMPLICIT_LINK_LIBRARIES})
-# set(CMAKE_Fortran_IMPLICIT_LINK_LIBRARIES "")
-# set(CMAKE_C_IMPLICIT_LINK_LIBRARIES "")
-# set(CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "")
-
-if(ENABLE_OPENMP)
- find_package(OpenMP REQUIRED)
- # Add OpenMP flags for C and Fortran
- # if(OpenMP_C_FOUND)
- # target_link_libraries(${BIN_NAME} PRIVATE OpenMP::OpenMP_C)
- # endif()
-
- if(OpenMP_Fortran_FOUND)
- target_link_libraries(${BIN_NAME} PRIVATE OpenMP::OpenMP_Fortran)
- endif()
-
- message(STATUS "OpenMP enabled for C and Fortran")
+if(M3DC1_ENABLE_ST)
+ target_compile_definitions(m3dc1_library PRIVATE USEST)
+endif()
+if(M3DC1_ENABLE_COMPLEX)
+ target_compile_definitions(m3dc1_library PRIVATE vectype=complex USECOMPLEX)
+else()
+ target_compile_definitions(m3dc1_library PRIVATE vectype=real)
+endif()
+if(M3DC1_ENABLE_OPENMP)
+ target_link_libraries(m3dc1_library PUBLIC OpenMP::OpenMP_Fortran)
+endif()
+if(M3DC1_ENABLE_PARTICLE)
+ target_compile_definitions(m3dc1_library PRIVATE USEPARTICLES)
+endif()
+if(M3DC1_ENABLE_ADAS)
+ target_compile_definitions(m3dc1_library PRIVATE USEADAS)
+endif()
+set_target_properties(m3dc1_library PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
+
+target_compile_options(m3dc1_library PUBLIC
+ # GNU
+ $<$:-cpp>
+ $<$:-fdefault-real-8>
+ $<$:-fdefault-double-8>
+ $<$:-fallow-argument-mismatch>
+ $<$:-ffree-line-length-none>
+
+ # Intel Classic
+ $<$:-fpp>
+ $<$:-r8>
+
+ # IntelLLVM (ifx)
+ $<$:-fpp>
+ $<$:-r8>
+
+ # LLVM Flang
+ $<$:-cpp>
+ $<$:-fdefault-real-8>
+ $<$:-fdefault-double-8>
+
+ # Cray Compiling Environment
+ $<$:-eZ>
+ $<$:-s>
+ $<$:real64>
+ $<$:-ef>
+)
+
+# --- Build main executable name per Makefile convention ---
+set(M3DC1_MAIN_NAME m3dc1)
+if(M3DC1_ENABLE_3D)
+ string(APPEND M3DC1_MAIN_NAME "_3d")
else()
- message(STATUS "OpenMP not enabled")
+ string(APPEND M3DC1_MAIN_NAME "_2d")
+endif()
+if(M3DC1_ENABLE_TRILINOS)
+ string(APPEND M3DC1_MAIN_NAME "_trilinos")
+endif()
+if(M3DC1_ENABLE_COMPLEX)
+ string(APPEND M3DC1_MAIN_NAME "_complex")
+endif()
+if(M3DC1_ENABLE_OPENMP)
+ string(APPEND M3DC1_MAIN_NAME "_omp")
+endif()
+if(M3DC1_ENABLE_ST)
+ string(APPEND M3DC1_MAIN_NAME "_st")
endif()
+if(M3DC1_ENABLE_PARTICLE)
+ string(APPEND M3DC1_MAIN_NAME "_pic")
+endif()
+
+add_executable(${M3DC1_MAIN_NAME} $)
+target_link_libraries(${M3DC1_MAIN_NAME} PRIVATE m3dc1_library)
-#find_package(PSPLINE)
-#add_executable(readgato polar.f90 readgato.f90)
-#target_link_libraries(readgato ${PSPLINE_LIBRARIES})
+add_custom_target(m3dc1 ALL
+ COMMAND ${CMAKE_COMMAND} -E create_symlink ${M3DC1_MAIN_NAME} m3dc1
+ DEPENDS ${M3DC1_MAIN_NAME}
+ WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+ COMMENT "Creating symlink: m3dc1 -> ${M3DC1_MAIN_NAME}")
-#add_executable(read_jsolver polar.f90 read_jsolver_exec.f90)
-#target_link_libraries(read_jsolver ${PSPLINE_LIBRARIES})
+add_executable(a2cc a2cc.f90)
+target_link_libraries(a2cc PUBLIC m3dc1_library)
-#add_executable(a2cc readaeqdsk.f90 a2cc.f90)
-#target_link_libraries(a2cc )
-#target_compile_definitions(a2cc PUBLIC ${DEFS})
+add_executable(mackenbach_profiles mackenbach_profiles.f90)
-#add_executable(kprad kprad.f90 kprad_test.f90)
+# add_executable(kprad kprad.f90 kprad_test.f90)
+add_executable(test_region test_region.f90)
+target_link_libraries(test_region PUBLIC m3dc1_library)
+if(M3DC1_ENABLE_PSPLINE)
+ find_package(PSPLINE REQUIRED)
+ add_executable(readgato readgato.f90)
+ target_link_libraries(readgato PUBLIC m3dc1_library PSPLINE::PSPLINE)
+
+ add_executable(read_jsolver read_jsolver.f90 polar.f90)
+ target_compile_definitions(read_jsolver PRIVATE READ_JSOLVER)
+ target_link_libraries(read_jsolver PUBLIC PSPLINE::PSPLINE)
+endif()
diff --git a/unstructured/M3Dmodules.f90 b/unstructured/M3Dmodules.f90
index 8c7e0c958..4745107cf 100644
--- a/unstructured/M3Dmodules.f90
+++ b/unstructured/M3Dmodules.f90
@@ -821,12 +821,7 @@ end module m3dc1_omp
!cj velocity equation profiling
module m3dc1_vel_prof
-#if PETSC_VERSION >= 39
-#include
use petscksp
-#else
-#include
-#endif
implicit none
- PetscLogStage stageA,stageS
+ integer :: stageA, stageS
end module m3dc1_vel_prof
diff --git a/unstructured/basic_mesh.f90 b/unstructured/basic_mesh.f90
index 77619454c..ea3dec8ae 100644
--- a/unstructured/basic_mesh.f90
+++ b/unstructured/basic_mesh.f90
@@ -287,33 +287,36 @@ end subroutine create_rectangular_mesh
subroutine load_mesh
+ use petscmat
+ use petscis
+ use petscao
+ use petscsys
use math
implicit none
-#include "finclude/petsc.h"
!#ifndef PETSC_31
-!#include "finclude/petscvec.h"
-!#include "finclude/petscmat.h"
-!#include "finclude/petscis.h"
+!#include
+!#include
+!#include
!#endif
-#include "finclude/petscis.h90"
+!#include
integer :: num_global_elements
type(node_type), allocatable :: global_node(:)
type(element_type), allocatable :: global_elm(:)
- Mat :: connectivity, adjacency
- MatPartitioning :: partitioning
+ type(tMat) :: connectivity, adjacency
+ type(tMatPartitioning) :: partitioning
integer :: i, j, k, ierr, rank, size, itri
integer :: min_node, max_node
integer, allocatable :: local_id(:)
logical, allocatable :: is_ghost(:)
- PetscInt, pointer :: idx(:)
+ integer, pointer :: idx(:)
integer, allocatable :: nodes_per_proc(:), nodes_per_proc_local(:)
integer :: new_id(1)
- IS :: global_numbering, node_distribution
- AO :: ordering
- PetscViewer :: pv
+ type(tIS) :: global_numbering, node_distribution
+ type(tAO) :: ordering
+ type(tPetscViewer) :: pv
bb(1) = 0.185
bb(2) = -1.45
@@ -386,7 +389,7 @@ subroutine load_mesh
call MatDestroy(adjacency, ierr)
call ISPartitioningToNumbering(node_distribution,global_numbering,ierr)
- call AOCreateBasicIS(global_numbering, PETSC_NULL, ordering, ierr)
+ call AOCreateBasicIS(global_numbering, PETSC_NULL_IS, ordering, ierr)
call PetscViewerASCIIOpen(PETSC_COMM_WORLD,'node_distribution',pv,ierr)
call ISView(node_distribution, pv, ierr)
@@ -405,11 +408,11 @@ subroutine load_mesh
allocate(nodes_per_proc_local(size), nodes_per_proc(size))
nodes_per_proc_local = 0
- call ISGetIndicesF90(node_distribution, idx, ierr)
+ call ISGetIndices(node_distribution, idx, ierr)
do i=1, num_local_nodes
nodes_per_proc_local(idx(i)+1) = nodes_per_proc_local(idx(i)+1) + 1
end do
- call ISRestoreIndicesF90(node_distribution, idx, ierr)
+ call ISRestoreIndices(node_distribution, idx, ierr)
call mpi_allreduce(nodes_per_proc_local, nodes_per_proc, size, &
MPI_INTEGER, MPI_SUM, MPI_COMM_WORLD, ierr)
diff --git a/unstructured/cmake/FindParmetis.cmake b/unstructured/cmake/FindParmetis.cmake
deleted file mode 100644
index 0224617d9..000000000
--- a/unstructured/cmake/FindParmetis.cmake
+++ /dev/null
@@ -1,46 +0,0 @@
-# - Try to find parmetis
-# Once done this will define
-# PARMETIS_FOUND - System has PARMETIS
-# PARMETIS_INCLUDE_DIRS - The PARMETIS include directories
-# PARMETIS_LIBRARIES - The libraries needed to use PARMETIS
-# PARMETIS_DEFINITIONS - Compiler switches required for using PARMETIS
-
-set(PARMETIS_PREFIX "" CACHE STRING "ParMETIS install directory")
-
-find_path(PARMETIS_INCLUDE_DIR parmetis.h PATHS "${PARMETIS_PREFIX}/include")
-if(NOT EXISTS "${PARMETIS_INCLUDE_DIR}")
- message(FATAL_ERROR "parmetis include dir not found")
-endif()
-
-find_path(METIS_INCLUDE_DIR metis.h PATHS "${PARMETIS_PREFIX}/include")
-if(NOT EXISTS "${METIS_INCLUDE_DIR}")
- message(FATAL_ERROR "metis include dir not found")
-endif()
-
-
-find_library(PARMETIS_LIBRARY parmetis PATHS "${PARMETIS_PREFIX}/lib")
-if(NOT EXISTS "${PARMETIS_LIBRARY}")
- message(FATAL_ERROR "parmetis library not found")
-endif()
-
-find_library(METIS_LIBRARY metis PATHS "${PARMETIS_PREFIX}/lib")
-if(NOT EXISTS "${METIS_LIBRARY}")
- message(FATAL_ERROR "metis library not found")
-endif()
-
-find_library(GK_LIBRARY GKlib PATHS "${PARMETIS_PREFIX}/lib")
-if(EXISTS "${GK_LIBRARY}")
- set(PARMETIS_LIBRARIES ${PARMETIS_LIBRARY} ${METIS_LIBRARY} ${GK_LIBRARY})
-else()
- set(PARMETIS_LIBRARIES ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
-endif()
-
-set(PARMETIS_INCLUDE_DIRS ${PARMETIS_INCLUDE_DIR} ${METIS_INCLUDE_DIR})
-
-include(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set PARMETIS_FOUND to TRUE
-# if all listed variables are TRUE
-find_package_handle_standard_args(Parmetis DEFAULT_MSG
- PARMETIS_LIBRARY METIS_LIBRARY PARMETIS_INCLUDE_DIR)
-
-mark_as_advanced(PARMETIS_INCLUDE_DIR PARMETIS_LIBRARY METIS_LIBRARY)
diff --git a/unstructured/cmake/FindScorec.cmake b/unstructured/cmake/FindScorec.cmake
deleted file mode 100644
index 4f6e8c45d..000000000
--- a/unstructured/cmake/FindScorec.cmake
+++ /dev/null
@@ -1,106 +0,0 @@
-# - Try to find SCOREC PUMI libraries
-# Once done this will define
-# SCOREC_FOUND - System has SCOREC
-# SCOREC_INCLUDE_DIRS - The SCOREC include directories
-# SCOREC_LIBRARIES - The libraries needed to use SCOREC
-# SCOREC_DEFINITIONS - Compiler switches required for using SCOREC
-#
-# This implementation assumes a SCOREC install has the following structure
-# VERSION/
-# include/*.h
-# lib/*.a
-
-macro(scorecLibCheck libs isRequired)
- foreach(lib ${libs})
- unset(scoreclib CACHE)
- find_library(scoreclib "${lib}" PATHS ${SCOREC_LIB_DIR})
- if(scoreclib MATCHES "^scoreclib-NOTFOUND$")
- if(${isRequired})
- message(FATAL_ERROR "SCOREC library ${lib} not found in ${SCOREC_LIB_DIR}")
- else()
- message("SCOREC library ${lib} not found in ${SCOREC_LIB_DIR}")
- endif()
- else()
- set("SCOREC_${lib}_FOUND" TRUE CACHE INTERNAL "SCOREC library present")
- set(SCOREC_LIBS ${SCOREC_LIBS} ${scoreclib})
- endif()
- endforeach()
-endmacro(scorecLibCheck)
-
-if(CMAKE_CXX_COMPILER_ID MATCHES "PGI")
- set(ENABLE_ZOLTAN OFF)
-endif()
-
-find_library(METIS_LIBRARY metis)
-if (NOT EXISTS "${METIS_LIBRARY}")
- message(FATAL ERROR "METIS library not found")
-endif()
-
-set(SCOREC_LIBS "")
-if(ENABLE_SIMMETRIX)
-set(SCOREC_LIB_NAMES
- pumi
- ma
- mds
- apf_sim
- apf_zoltan
- apf
- parma
- gmi
- gmi_sim
- mth
- pcu
- sam
- spr
- crv
- lion
- ph
- )
-else()
-set(SCOREC_LIB_NAMES
-pumi
-crv
-ph
-sam
-spr
-ma
-apf_zoltan
-parma
-mds
-apf
-lion
-mth
-gmi
-pcu
- )
-endif()
-scorecLibCheck("${SCOREC_LIB_NAMES}" TRUE)
-
-find_path(SCOREC_INCLUDE_DIR
- NAMES apf.h PCU.h ma.h
- PATHS ${SCOREC_INCLUDE_DIR})
-if(NOT EXISTS "${SCOREC_INCLUDE_DIR}")
- message(FATAL_ERROR "SCOREC include dir not found")
-endif()
-
-string(REGEX REPLACE
- "/include$" ""
- SCOREC_INSTALL_DIR
- "${SCOREC_INCLUDE_DIR}")
-
-set(SCOREC_LIBRARIES ${SCOREC_LIBS} ${ZOLTAN_LIBRARY} ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
-set(SCOREC_INCLUDE_DIRS ${SCOREC_INCLUDE_DIR})
-
-include(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set PARMETIS_FOUND to TRUE
-# if all listed variables are TRUE
-find_package_handle_standard_args(Scorec DEFAULT_MSG
- SCOREC_LIBS SCOREC_INCLUDE_DIR)
-
-mark_as_advanced(SCOREC_INCLUDE_DIR SCOREC_LIBS ${ZOLTAN_LIBRARY} ${PARMETIS_LIBRARY} ${METIS_LIBRARY})
-
-set(SCOREC_LINK_LIBS "")
-foreach(lib ${SCOREC_LIB_NAMES})
- set(SCOREC_LINK_LIBS "${SCOREC_LINK_LIBS} -l${lib}")
-endforeach()
-
diff --git a/unstructured/cmake/FindZoltan.cmake b/unstructured/cmake/FindZoltan.cmake
deleted file mode 100644
index 24d8ad654..000000000
--- a/unstructured/cmake/FindZoltan.cmake
+++ /dev/null
@@ -1,31 +0,0 @@
-# - Try to find zoltan
-# Once done this will define
-# ZOLTAN_FOUND - System has ZOLTAN
-# ZOLTAN_INCLUDE_DIRS - The ZOLTAN include directories
-# ZOLTAN_LIBRARIES - The libraries needed to use ZOLTAN
-# ZOLTAN_DEFINITIONS - Compiler switches required for using ZOLTAN
-
-set(ZOLTAN_PREFIX "${ZOLTAN_PREFIX_DEFAULT}" CACHE STRING "Zoltan install directory")
-if(ZOLTAN_PREFIX)
- message(STATUS "ZOLTAN_PREFIX ${ZOLTAN_PREFIX}")
-endif()
-
-find_path(ZOLTAN_INCLUDE_DIR zoltan.h PATHS "${ZOLTAN_PREFIX}/include")
-
-find_library(ZOLTAN_LIBRARY zoltan PATHS "${ZOLTAN_PREFIX}/lib")
-
-set(ZOLTAN_LIBRARIES ${ZOLTAN_LIBRARY} )
-set(ZOLTAN_INCLUDE_DIRS ${ZOLTAN_INCLUDE_DIR} )
-
-find_package(Parmetis MODULE REQUIRED)
-
-include(FindPackageHandleStandardArgs)
-# handle the QUIETLY and REQUIRED arguments and set ZOLTAN_FOUND to TRUE
-# if all listed variables are TRUE
-find_package_handle_standard_args(
- Zoltan
- DEFAULT_MSG
- ZOLTAN_LIBRARY ZOLTAN_INCLUDE_DIR
-)
-
-mark_as_advanced(ZOLTAN_INCLUDE_DIR ZOLTAN_LIBRARY )
diff --git a/unstructured/coils.f90 b/unstructured/coils.f90
index 917f1dd42..267039539 100644
--- a/unstructured/coils.f90
+++ b/unstructured/coils.f90
@@ -12,16 +12,15 @@ module coils
!
! reads coil and current data from file
!======================================================
- subroutine load_coils(xc, zc, ic, numcoils, coil_filename, current_filename, &
- coil_mask, filaments)
- use math
- use read_ascii
+ subroutine load_coils(xc, zc, ic, numcoils, coil_filename, current_filename, &
+ coil_mask, filaments)
+ use math
+ use read_ascii
+ use mpi
- implicit none
+ implicit none
- include 'mpif.h'
-
- real, intent(out), dimension(maxfilaments) :: xc, zc ! coordinates of each coil
+ real, intent(out), dimension(maxfilaments) :: xc, zc ! coordinates of each coil
complex, intent(out), dimension(maxfilaments) :: ic ! current in each coil
integer, intent(out) :: numcoils ! number of coils read
character*(*) :: coil_filename, current_filename ! input files
@@ -160,15 +159,13 @@ end subroutine load_coils
subroutine field_from_coils(xc, zc, ic, nc, f, ipole, ierr)
use basic
use field
- use mesh_mod
- use m3dc1_nint
- use newvar_mod
-
- implicit none
+ use mesh_mod
+ use m3dc1_nint
+ use newvar_mod
- include 'mpif.h'
+ implicit none
- real, intent(in), dimension(nc) :: xc, zc ! array of coil positions
+ real, intent(in), dimension(nc) :: xc, zc ! array of coil positions
complex, intent(in), dimension(nc) :: ic ! array of coil currents
integer, intent(in) :: nc ! number of coils
type(field_type), intent(inout) :: f ! poloidal flux field
diff --git a/unstructured/diagnostics.f90 b/unstructured/diagnostics.f90
index c45a335a3..07f7a22a3 100644
--- a/unstructured/diagnostics.f90
+++ b/unstructured/diagnostics.f90
@@ -7,6 +7,8 @@
module diagnostics
+ use mpi
+
implicit none
real :: tflux0
@@ -129,7 +131,6 @@ subroutine distribute_timings
implicit none
- include 'mpif.h'
integer :: ier
integer, parameter :: num_scalars = 14
real, dimension(num_scalars) :: vin, vout
@@ -296,8 +297,6 @@ subroutine distribute_scalars()
implicit none
- include 'mpif.h'
-
integer, parameter :: num_scalars = 82
integer :: ier
double precision, dimension(num_scalars) :: temp, temp2
@@ -509,8 +508,6 @@ subroutine evaluate(x,phi,z,ans,fin,itri,ierr)
implicit none
- include 'mpif.h'
-
integer, intent(inout) :: itri
real, intent(in) :: x, phi, z
type(field_type), intent(in) :: fin
@@ -725,8 +722,6 @@ subroutine calculate_scalars()
implicit none
- include 'mpif.h'
-
integer :: itri, numelms, def_fields, ier
integer :: is_edge(3) ! is inode on boundary
real :: n(2,3),tpifac,tpirzero, t0
@@ -1205,8 +1200,6 @@ subroutine calculate_Lor_vol()
implicit none
- include 'mpif.h'
-
integer :: itri, numelms, ier
integer :: is_edge(3) ! is inode on boundary
real :: tpifac,tpirzero
@@ -1365,8 +1358,6 @@ subroutine magaxis(xguess,zguess,psi,psim,imethod,ier)
implicit none
- include 'mpif.h'
-
real, intent(inout) :: xguess, zguess
type(field_type), intent(in) :: psi
integer, intent(in) :: imethod
@@ -1576,8 +1567,6 @@ subroutine te_max(xguess,zguess,te,tem,imethod,ier)
implicit none
- include 'mpif.h'
-
real, intent(inout) :: xguess, zguess
type(field_type), intent(in) :: te
integer, intent(in) :: imethod
@@ -1777,8 +1766,6 @@ subroutine te_max2(xguess,zguess,te,tem,imethod,ier)
implicit none
- include 'mpif.h'
-
real, intent(inout) :: xguess, zguess
type(field_type), intent(in) :: te
integer, intent(in) :: imethod
@@ -1836,8 +1823,6 @@ subroutine te_max3(xguess,zguess,te,tem,imethod,ier)
implicit none
- include 'mpif.h'
-
real, intent(inout) :: xguess, zguess
type(field_type), intent(in) :: te
real, intent(out) :: tem
@@ -1971,8 +1956,6 @@ subroutine te_max4(te,tem,ilin,ier)
implicit none
- include 'mpif.h'
-
type(field_type), intent(in) :: te
integer, intent(in) :: ilin
real, intent(out) :: tem
@@ -2038,8 +2021,6 @@ subroutine lcfs(psi, test_wall, findx)
implicit none
- include 'mpif.h'
-
type(field_type), intent(in) :: psi
logical, intent(in), optional :: test_wall
logical, intent(in), optional :: findx
@@ -2368,7 +2349,6 @@ subroutine calculate_ke()
use math
implicit none
- include 'mpif.h'
integer :: itri, numelms, def_fields
real :: ke_N, ketotal, fac
integer :: ier, k, l, numnodes, N, icounter_t
@@ -2735,7 +2715,6 @@ subroutine calculate_bh()
use boundary_conditions
use math
implicit none
- include 'mpif.h'
integer :: itri, numelms, def_fields
real:: bh_N, bhtotal, fac
integer :: ier, k, l, numnodes, N, icounter_t
@@ -3216,8 +3195,6 @@ subroutine te_max_dev(xguess,zguess,te,tem,imethod,ier)
implicit none
- include 'mpif.h'
-
real, intent(inout) :: xguess, zguess
type(field_type), intent(in) :: te
real, intent(out) :: tem
@@ -3490,8 +3467,6 @@ subroutine phi_int(x,z,ans,fin,itri,ierr)
implicit none
- include 'mpif.h'
-
integer, intent(inout) :: itri
real, intent(in) :: x, z
type(field_type), intent(in) :: fin
diff --git a/unstructured/error_estimate.f90 b/unstructured/error_estimate.f90
index 8f1f25b87..5f7fe7fa8 100644
--- a/unstructured/error_estimate.f90
+++ b/unstructured/error_estimate.f90
@@ -1,6 +1,6 @@
module error_estimate
+ use mpi
use m3dc1_nint
-#include "mpif.h"
integer, parameter :: EOP_1 = 1
integer, parameter :: EOP_DX = 2
integer, parameter :: EOP_DY = 3
diff --git a/unstructured/gradshafranov.f90 b/unstructured/gradshafranov.f90
index ed13376ed..3742e8578 100644
--- a/unstructured/gradshafranov.f90
+++ b/unstructured/gradshafranov.f90
@@ -126,6 +126,7 @@ subroutine gradshafranov_per()
end subroutine gradshafranov_per
subroutine coil_feedback(itnum)
+ use mpi
use basic
use arrays
use coils
@@ -133,7 +134,6 @@ subroutine coil_feedback(itnum)
implicit none
- include 'mpif.h'
integer, intent(in) :: itnum
@@ -1093,6 +1093,7 @@ end subroutine extend_pressure
!============================================================
subroutine gradshafranov_solve
+ use mpi
use math
use mesh_mod
@@ -1107,7 +1108,6 @@ subroutine gradshafranov_solve
use model
implicit none
- include 'mpif.h'
type(field_type) :: b1vecini_vec, b2vecini_vec
type(field_type) :: b3vecini_vec, b4vecini_vec
@@ -1814,6 +1814,7 @@ subroutine gradshafranov_solve
end subroutine gradshafranov_solve
subroutine calculate_error(error, error2, psinew)
+ use mpi
use basic
use field
use boundary_conditions
@@ -1821,7 +1822,6 @@ subroutine calculate_error(error, error2, psinew)
implicit none
- include 'mpif.h'
real, intent(out) :: error, error2
type(field_type), intent(in) :: psinew
@@ -1883,6 +1883,7 @@ end subroutine calculate_error
! constrain solution to have the specified current, etc.
!============================================================
subroutine calculate_gamma(g2, g3, g4)
+ use mpi
use basic
use mesh_mod
use arrays
@@ -1891,7 +1892,6 @@ subroutine calculate_gamma(g2, g3, g4)
implicit none
- include 'mpif.h'
real, intent(out) :: g2, g3, g4
@@ -2009,6 +2009,7 @@ end subroutine calculate_gamma
! sets jout_i =
! ===========================================================
subroutine deltafun(x,z,val,jout)
+ use mpi
use mesh_mod
use basic
@@ -2019,7 +2020,6 @@ subroutine deltafun(x,z,val,jout)
implicit none
- include 'mpif.h'
type(element_data) :: d
real, intent(in) :: x, z, val
@@ -2080,6 +2080,7 @@ end subroutine deltafun
! ===========================================================
subroutine gaussianfun(x,z,val,denom,jout)
+ use mpi
use mesh_mod
use basic
@@ -2090,7 +2091,6 @@ subroutine gaussianfun(x,z,val,denom,jout)
implicit none
- include 'mpif.h'
real, intent(in) :: x, z, val, denom
type(field_type), intent(inout) :: jout
@@ -2371,6 +2371,7 @@ end subroutine fundef
subroutine fundef2(error)
+ use mpi
use basic
use mesh_mod
@@ -2382,7 +2383,6 @@ subroutine fundef2(error)
implicit none
- include 'mpif.h'
real, intent(out) :: error
@@ -2878,12 +2878,12 @@ end subroutine write_profile
! calculates the fractional error in the GS solution
!=============================================================
subroutine calculate_gs_error(error)
+ use mpi
use basic
use m3dc1_nint
implicit none
- include 'mpif.h'
real, intent(out) :: error
diff --git a/unstructured/hdf5_output.f90 b/unstructured/hdf5_output.f90
index f20824d23..0b53643e3 100644
--- a/unstructured/hdf5_output.f90
+++ b/unstructured/hdf5_output.f90
@@ -30,12 +30,12 @@ end subroutine hdf5_flush
! hdf5_initialize
! ===============
subroutine hdf5_initialize(restart, error)
+ use mpi
use hdf5
use basic
implicit none
- include 'mpif.h'
logical, intent(in) :: restart ! if true, do not overwrite file
integer, intent(out) :: error
diff --git a/unstructured/init_common.f90 b/unstructured/init_common.f90
index bbce4f563..14cf48f29 100644
--- a/unstructured/init_common.f90
+++ b/unstructured/init_common.f90
@@ -669,6 +669,7 @@ end subroutine kinetic_eq
#endif
subroutine nre_eq
+ use mpi
use basic
use arrays
use diagnostics
@@ -679,7 +680,6 @@ subroutine nre_eq
use pellet
implicit none
- include 'mpif.h'
type(field_type) :: nre_vec
integer :: itri, numelms, def_fields
diff --git a/unstructured/input.f90 b/unstructured/input.f90
index 7279b0859..806dee06a 100644
--- a/unstructured/input.f90
+++ b/unstructured/input.f90
@@ -97,12 +97,12 @@ end subroutine add_group
! reads input namelist
!=========================
subroutine input
+ use mpi
use basic
implicit none
integer :: ierr
-#include "mpif.h"
call set_defaults
@@ -1349,16 +1349,8 @@ subroutine validate_input
use resistive_wall
use kprad_m3dc1
-#if PETSC_VERSION >= 38
use petsc
implicit none
-#elif PETSC_VERSION >= 36
- implicit none
-#include "petsc/finclude/petsc.h"
-#else
- implicit none
-#include "finclude/petsc.h"
-#endif
integer :: ier,i
real :: de
@@ -1871,4 +1863,3 @@ subroutine validate_input
end if
end subroutine validate_input
-
diff --git a/unstructured/iterdb.f90 b/unstructured/iterdb.f90
index 9a93645eb..38260d88e 100644
--- a/unstructured/iterdb.f90
+++ b/unstructured/iterdb.f90
@@ -28,10 +28,10 @@ module iterdb
contains
subroutine load_iterdb(filename, ierr)
+ use mpi
implicit none
- include 'mpif.h'
character(len=*), intent(in) :: filename
integer, intent(out) :: ierr
diff --git a/unstructured/kprad.f90 b/unstructured/kprad.f90
index b3528ca96..d0071f05b 100644
--- a/unstructured/kprad.f90
+++ b/unstructured/kprad.f90
@@ -83,9 +83,9 @@ subroutine kprad_allocate(Z)
end subroutine kprad_allocate
subroutine kprad_rebase_dt()
+ use mpi
implicit none
- include 'mpif.h'
integer :: ier
real :: temp
diff --git a/unstructured/kprad_m3dc1.f90 b/unstructured/kprad_m3dc1.f90
index d14a0bfa4..f4752cc34 100644
--- a/unstructured/kprad_m3dc1.f90
+++ b/unstructured/kprad_m3dc1.f90
@@ -274,11 +274,10 @@ subroutine kprad_advect(dti)
use boundary_conditions
use metricterms_new
use sparse
+ use mpi
implicit none
- include 'mpif.h'
-
real, intent(in) :: dti
type(matrix_type) :: nmat_lhs, nmat_rhs
type(field_type) :: rhs
@@ -730,11 +729,10 @@ subroutine read_lp_source(filename, ierr)
use pellet
use newvar_mod
use math
+ use mpi
implicit none
- include 'mpif.h'
-
integer :: mpierr
integer, intent(out) :: ierr
character(len=*), intent(in) :: filename
@@ -873,11 +871,10 @@ subroutine deltafuns(n,x,phi,z,m,val,jout, ier)
use field
use m3dc1_nint
use math
+ use mpi
implicit none
- include 'mpif.h'
-
integer, intent(in) :: n, m
real, intent(in), dimension(n) :: x, phi, z
real, intent(in), dimension(n,m) :: val
diff --git a/unstructured/kprad_test.f90 b/unstructured/kprad_test.f90
index 062026da4..41ed3924b 100644
--- a/unstructured/kprad_test.f90
+++ b/unstructured/kprad_test.f90
@@ -4,8 +4,9 @@ program main
implicit none
integer :: zimp, i, j, ntimemax,ntime,nzones, ierr
real :: dts, facimp, time
- real, allocatable, dimension(:) :: ne,te,pbrem,dw_brem
- real, allocatable, dimension(:,:) :: imp_rad,dw_rad,nz
+ real, allocatable, dimension(:) :: ne, te, den, ti, p, pbrem, dw_brem
+ real, allocatable, dimension(:,:) :: imp_rad, dw_rad, dw_ion, dw_reck, &
+ dw_recp, nz, source
open(unit=10,file='rad.txt')
@@ -13,18 +14,31 @@ program main
ZIMP = 10 ! NEON
! ZIMP = 18 ! Argon
+ ikprad = 1
+ ikprad_min_option = 1
+ ikprad_max_dt = 0
+ kprad_max_dt = -1.
+ ikprad_evolve_internal = 0
+
+ call kprad_allocate(zimp)
call kprad_atomic_data_sub(zimp, ierr)
dts = 1.e-7
ntimemax = 100
nzones = 1 ! one zone
j = 1
- allocate(ne(nzones),te(nzones))
+ allocate(ne(nzones), te(nzones), den(nzones), ti(nzones), p(nzones))
ne(j) = 1.e14 ! density in cm-3
te(j) = 1000. ! temp in ev
+ den(j) = ne(j)
+ ti(j) = te(j)
+ p(j) = ne(j)*te(j)*1.6022e-12
allocate(nz(nzones,0:zimp))
allocate(pbrem(nzones), imp_rad(nzones,zimp+1), &
- dw_brem(nzones), dw_rad(nzones,zimp+1))
+ dw_brem(nzones), dw_rad(nzones,zimp+1), &
+ dw_ion(nzones,0:zimp), dw_reck(nzones,0:zimp), &
+ dw_recp(nzones,0:zimp), source(nzones,0:zimp))
+ source = 0.
! initialize
facimp = .01
@@ -38,8 +52,9 @@ program main
do ntime=1, ntimemax
time = time + dts
- call kprad_advance_densities(dts, nzones, zimp, ne, te, nz, &
- dw_rad, dw_brem)
+ call kprad_advance_densities(dts, nzones, zimp, p, ne, te, den, ti, &
+ nz, dw_rad, dw_brem, dw_ion, dw_reck, &
+ dw_recp, source)
! calculate power from radiated energy
pbrem = dw_brem / dts
@@ -54,7 +69,8 @@ program main
call kprad_deallocate()
- deallocate(nz, pbrem, imp_rad, dw_brem, dw_rad)
+ deallocate(ne, te, den, ti, p, nz, pbrem, imp_rad, dw_brem, dw_rad, &
+ dw_ion, dw_reck, dw_recp, source)
stop
diff --git a/unstructured/matrix_mod.F90 b/unstructured/matrix_mod.F90
new file mode 100644
index 000000000..8933c67c3
--- /dev/null
+++ b/unstructured/matrix_mod.F90
@@ -0,0 +1,14 @@
+! matrix_mod.F90 — thin wrapper that selects the matrix backend at compile time.
+! CMake defines exactly one of the following:
+! M3DC1_MATRIX_IS_SCOREC → use scorec_matrix_mod
+! M3DC1_MATRIX_IS_PETSC → use petsc_matrix_mod
+
+module matrix_mod
+#ifdef M3DC1_MATRIX_IS_SCOREC
+ use scorec_matrix_mod
+#elif defined(M3DC1_MATRIX_IS_PETSC)
+ use petsc_matrix_mod
+#else
+ use scorec_matrix_mod
+#endif
+end module matrix_mod
diff --git a/unstructured/mesh_mod.F90 b/unstructured/mesh_mod.F90
new file mode 100644
index 000000000..55b61522c
--- /dev/null
+++ b/unstructured/mesh_mod.F90
@@ -0,0 +1,14 @@
+! mesh_mod.F90 — thin wrapper that selects the mesh backend at compile time.
+! CMake defines exactly one of the following:
+! M3DC1_MESH_IS_SCOREC → use scorec_mesh_mod
+! M3DC1_MESH_IS_BASIC → use basic_mesh_mod
+
+module mesh_mod
+#ifdef M3DC1_MESH_IS_SCOREC
+ use scorec_mesh_mod
+#elif defined(M3DC1_MESH_IS_BASIC)
+ use basic_mesh_mod
+#else
+ use scorec_mesh_mod
+#endif
+end module mesh_mod
diff --git a/unstructured/newpar.f90 b/unstructured/newpar.f90
index d5f1976df..ecc2dc37e 100644
--- a/unstructured/newpar.f90
+++ b/unstructured/newpar.f90
@@ -34,16 +34,8 @@ Program Reducedquintic
use openacc
#endif
-#if PETSC_VERSION >= 38
use petsc
implicit none
-#elif PETSC_VERSION >= 36
- implicit none
-#include "petsc/finclude/petsc.h"
-#else
- implicit none
-#include "finclude/petsc.h"
-#endif
integer :: ier, i, adapt_flag
real :: tstart, tend, dtsave, t_solve, t_compute
@@ -582,16 +574,8 @@ subroutine safestop(iarg)
use particles
use resistive_wall
-#if PETSC_VERSION >= 38
use petsc
implicit none
-#elif PETSC_VERSION >= 36
- implicit none
-#include "petsc/finclude/petsc.h"
-#else
- implicit none
-#include "finclude/petsc.h"
-#endif
integer, intent(in) :: iarg
integer :: ier
@@ -1030,13 +1014,13 @@ end subroutine rotation
! populates the *tri arrays
!============================================================
subroutine tridef
+ use mpi
use basic
use math
use mesh_mod
implicit none
- include 'mpif.h'
type(element_data) :: d
integer :: itri, i, j, k, ii, jj, numelms, numnodes, ndofs, ierr
@@ -1599,12 +1583,12 @@ subroutine calculate_qdfac(itri, z)
end subroutine calculate_qdfac
subroutine print_normal_curv()
+ use mpi
! use mpi
use basic
use mesh_mod
implicit none
- include 'mpif.h'
integer :: ierr, i, icounter_t, numnodes
integer :: izone, izonedim
diff --git a/unstructured/output.f90 b/unstructured/output.f90
index 3e6557ef0..fb1a2e158 100644
--- a/unstructured/output.f90
+++ b/unstructured/output.f90
@@ -59,6 +59,7 @@ end subroutine finalize_output
!
! ======================================================================
subroutine marker
+ use mpi
use basic
use hdf5_output
use diagnostics
@@ -66,7 +67,6 @@ subroutine marker
implicit none
- include 'mpif.h'
integer :: ier,i
call mark_fields(0);
@@ -80,6 +80,7 @@ end subroutine marker
! writes output and restart files
! ======================================================================
subroutine output
+ use mpi
use basic
use hdf5_output
use diagnostics
@@ -89,7 +90,6 @@ subroutine output
implicit none
- include 'mpif.h'
integer :: ier
real :: tstart, tend, diff, gamma_std, gamma_mean
@@ -665,13 +665,13 @@ end subroutine hdf5_write_timings
! hdf5_write_time_slice
! =====================
subroutine hdf5_write_time_slice(equilibrium, error)
+ use mpi
use hdf5
use hdf5_output
use basic
implicit none
- include 'mpif.h'
integer, intent(out) :: error
integer, intent(in) :: equilibrium
diff --git a/unstructured/particle.f90 b/unstructured/particle.f90
index 2041b5352..b220537c1 100644
--- a/unstructured/particle.f90
+++ b/unstructured/particle.f90
@@ -136,9 +136,9 @@ module particles
! Note: any changes to the "particle" user-defined datatype must be reflected
! in the definitions of pnvars, pblklen, ptyps, and pdspls below.
subroutine define_mpi_particle(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: pnvars = 15
@@ -189,9 +189,9 @@ subroutine define_mpi_particle(ierr)
end subroutine define_mpi_particle
subroutine define_mpi_elfield(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: pnvars = 24
@@ -264,13 +264,13 @@ end subroutine define_mpi_elfield
!---------------------------------------------------------------------------
subroutine particle_test
+ use mpi
use basic
use diagnostics
use auxiliary_fields
use m3dc1_nint
use arrays
implicit none
- include 'mpif.h'
integer, parameter :: trunit = 120
real :: tstart, tend
@@ -382,6 +382,7 @@ real function tri_area(x1, z1, x2, z2, x3, z3)
end function tri_area
!---------------------------------------------------------------------------
subroutine init_particles(lrestart, ierr)
+ use mpi
use basic
use arrays
use hdf5_output
@@ -392,7 +393,6 @@ subroutine init_particles(lrestart, ierr)
use boundary_conditions
implicit none
- include 'mpif.h'
logical, intent(in) :: lrestart
integer, intent(out) :: ierr
@@ -1914,13 +1914,13 @@ subroutine fdot(x, v, w, dxdt, dvdt, dwdt, dEpdt, itri, kel, f00, ierr, sps, B00
end subroutine fdot
subroutine particle_scaleback(scalefac)
+ use mpi
use basic
use field
use arrays
use mesh_mod
implicit none
- include 'mpif.h'
vectype, intent(in) :: scalefac
integer :: ipart, ielm, ierr
@@ -1951,10 +1951,10 @@ subroutine particle_scaleback(scalefac)
end subroutine particle_scaleback
!---------------------------------------------------------------------------
subroutine delete_particle(exchange)
+ use mpi
use basic
use diagnostics
implicit none
- include 'mpif.h'
logical, intent(in) :: exchange
!integer, intent(in) :: ipart
@@ -2029,11 +2029,11 @@ end subroutine delete_particle
!---------------------------------------------------------------------------
subroutine particle_step(pdt)
+ use mpi
use basic
use diagnostics
use auxiliary_fields
implicit none
- include 'mpif.h'
real, intent(in) :: pdt
@@ -2093,11 +2093,11 @@ subroutine particle_step(pdt)
end subroutine particle_step
!---------------------------------------------------------------------------
subroutine update_particle_pressure
+ use mpi
use basic
use arrays
use diagnostics
implicit none
- include 'mpif.h'
real :: tstart, tend
integer :: ierr
@@ -2127,9 +2127,9 @@ subroutine update_particle_pressure
end subroutine update_particle_pressure
!---------------------------------------------------------------------------
subroutine finalize_particles
+ use mpi
use arrays
implicit none
- include 'mpif.h'
!integer :: nelms, ielm
integer :: ierr
@@ -2631,11 +2631,11 @@ subroutine update_geom_terms_st(gh, fh, ic2)
end subroutine update_geom_terms_st
!---------------------------------------------------------------------------
subroutine get_field_coefs(eq)
+ use mpi
use arrays
use basic
use auxiliary_fields
implicit none
- include 'mpif.h'
!type(elfield), intent(out) :: fh !Field handle
integer, intent(in) :: eq
@@ -3274,12 +3274,12 @@ subroutine evalf0(x, vpar, vperp, fh, gh, sps, f0, gradcoef, df0de, df0dxi)
end subroutine evalf0
!---------------------------------------------------------------------------
subroutine particle_pressure_rhs
+ use mpi
use basic
use arrays
use math
use m3dc1_nint
implicit none
- include 'mpif.h'
intrinsic matmul
real, dimension(dofs_per_element, coeffs_per_element) :: cl
@@ -3612,11 +3612,11 @@ end subroutine solve_pi_tensor
!---------------------------------------------------------------------------
! Dump particle data for current timeslice using parallel HDF5.
subroutine hdf5_write_particles(ierr)
+ use mpi
use basic
use hdf5_output
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
@@ -3825,11 +3825,11 @@ end subroutine hdf5_write_particles
!---------------------------------------------------------------------------
!Read stored HDF5 particle data in parallel.
subroutine hdf5_read_particles(filename, ierr)
+ use mpi
use basic
use diagnostics
use hdf5_output
implicit none
- include 'mpif.h'
character(len=*), intent(in) :: filename
integer, intent(out) :: ierr
diff --git a/unstructured/particle_com.f90 b/unstructured/particle_com.f90
index 126f274f0..0268ff99e 100644
--- a/unstructured/particle_com.f90
+++ b/unstructured/particle_com.f90
@@ -136,9 +136,9 @@ module particles
! Note: any changes to the "particle" user-defined datatype must be reflected
! in the definitions of pnvars, pblklen, ptyps, and pdspls below.
subroutine define_mpi_particle(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: pnvars = 15
@@ -189,9 +189,9 @@ subroutine define_mpi_particle(ierr)
end subroutine define_mpi_particle
subroutine define_mpi_elfield(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: pnvars = 24
@@ -264,13 +264,13 @@ end subroutine define_mpi_elfield
!---------------------------------------------------------------------------
subroutine particle_test
+ use mpi
use basic
use diagnostics
use auxiliary_fields
use m3dc1_nint
use arrays
implicit none
- include 'mpif.h'
integer, parameter :: trunit = 120
real :: tstart, tend
@@ -376,6 +376,7 @@ real function tri_area(x1, z1, x2, z2, x3, z3)
end function tri_area
!---------------------------------------------------------------------------
subroutine init_particles(lrestart, ierr)
+ use mpi
use basic
use arrays
use hdf5_output
@@ -386,7 +387,6 @@ subroutine init_particles(lrestart, ierr)
use boundary_conditions
implicit none
- include 'mpif.h'
logical, intent(in) :: lrestart
integer, intent(out) :: ierr
@@ -1879,13 +1879,13 @@ subroutine fdot(x, v, w, dxdt, dvdt, dwdt, dEpdt, itri, kel, f00, ierr, sps, B00
end subroutine fdot
subroutine particle_scaleback(scalefac)
+ use mpi
use basic
use field
use arrays
use mesh_mod
implicit none
- include 'mpif.h'
vectype, intent(in) :: scalefac
integer :: ipart, ielm, ierr
@@ -1916,10 +1916,10 @@ subroutine particle_scaleback(scalefac)
end subroutine particle_scaleback
!---------------------------------------------------------------------------
subroutine delete_particle(exchange)
+ use mpi
use basic
use diagnostics
implicit none
- include 'mpif.h'
logical, intent(in) :: exchange
!integer, intent(in) :: ipart
@@ -1994,11 +1994,11 @@ end subroutine delete_particle
!---------------------------------------------------------------------------
subroutine particle_step(pdt)
+ use mpi
use basic
use diagnostics
use auxiliary_fields
implicit none
- include 'mpif.h'
real, intent(in) :: pdt
@@ -2057,11 +2057,11 @@ subroutine particle_step(pdt)
end subroutine particle_step
!---------------------------------------------------------------------------
subroutine update_particle_pressure
+ use mpi
use basic
use arrays
use diagnostics
implicit none
- include 'mpif.h'
real :: tstart, tend
integer :: ierr
@@ -2091,9 +2091,9 @@ subroutine update_particle_pressure
end subroutine update_particle_pressure
!---------------------------------------------------------------------------
subroutine finalize_particles
+ use mpi
use arrays
implicit none
- include 'mpif.h'
!integer :: nelms, ielm
integer :: ierr
@@ -2595,11 +2595,11 @@ subroutine update_geom_terms_st(gh, fh, ic2)
end subroutine update_geom_terms_st
!---------------------------------------------------------------------------
subroutine get_field_coefs(eq)
+ use mpi
use arrays
use basic
use auxiliary_fields
implicit none
- include 'mpif.h'
!type(elfield), intent(out) :: fh !Field handle
integer, intent(in) :: eq
@@ -3301,12 +3301,12 @@ subroutine evalf0(x, vpar, vperp, fh, gh, sps, B0, f0, df0dpsi, df0de, df0dxi)
end subroutine evalf0
!---------------------------------------------------------------------------
subroutine particle_pressure_rhs
+ use mpi
use basic
use arrays
use math
use m3dc1_nint
implicit none
- include 'mpif.h'
intrinsic matmul
real, dimension(dofs_per_element, coeffs_per_element) :: cl
@@ -3628,11 +3628,11 @@ end subroutine solve_pi_tensor
!---------------------------------------------------------------------------
! Dump particle data for current timeslice using parallel HDF5.
subroutine hdf5_write_particles(ierr)
+ use mpi
use basic
use hdf5_output
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
@@ -3846,11 +3846,11 @@ end subroutine hdf5_write_particles
!---------------------------------------------------------------------------
!Read stored HDF5 particle data in parallel.
subroutine hdf5_read_particles(filename, ierr)
+ use mpi
use basic
use diagnostics
use hdf5_output
implicit none
- include 'mpif.h'
character(len=*), intent(in) :: filename
integer, intent(out) :: ierr
diff --git a/unstructured/pellet.f90 b/unstructured/pellet.f90
index bd3aa3b77..6bc1dd497 100644
--- a/unstructured/pellet.f90
+++ b/unstructured/pellet.f90
@@ -306,12 +306,12 @@ end function pellet_distribution
! Keep track if a pellet has left the plasma domain and should no longer
! deposit material
subroutine pellet_domain
+ use mpi
use basic
implicit none
- include 'mpif.h'
integer :: j
integer :: itri, iz, izone, ier
diff --git a/unstructured/petsc_matrix.f90 b/unstructured/petsc_matrix.f90
index b0bd32c41..bac3ed3de 100644
--- a/unstructured/petsc_matrix.f90
+++ b/unstructured/petsc_matrix.f90
@@ -4,8 +4,8 @@ module petsc_matrix_mod
implicit none
-#include "finclude/petscmatdef.h"
-#include "finclude/petsckspdef.h"
+!#include
+!#include
type :: petsc_matrix
Mat :: data
@@ -107,9 +107,9 @@ subroutine petsc_matrix_create(mat, m, n, icomplex, lhs)
use mesh_mod
use vector_mod
implicit none
-#include "finclude/petsc.h"
+!#include
!#ifndef PETSC_31
-!#include "finclude/petscmat.h"
+!#include
!#endif
type(petsc_matrix) :: mat
@@ -247,7 +247,7 @@ subroutine petsc_matrix_insert_real(mat,val,i,j,iop)
use vector_mod
use mesh_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_matrix), intent(in) :: mat
real, intent(in) :: val
@@ -282,7 +282,7 @@ end subroutine petsc_matrix_insert_real
subroutine petsc_matrix_insert_complex(mat,val,i,j,iop)
use vector_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_matrix), intent(in) :: mat
complex, intent(in) :: val
@@ -334,7 +334,7 @@ end subroutine petsc_matrix_solve
!====================================================================
subroutine petsc_matrix_finalize(mat)
implicit none
-#include "finclude/petscmat.h"
+!#include
type(petsc_matrix) :: mat
integer :: ierr
@@ -349,7 +349,7 @@ end subroutine petsc_matrix_finalize
!====================================================================
subroutine petsc_matrix_flush(mat)
implicit none
-#include "finclude/petscmat.h"
+!#include
type(petsc_matrix) :: mat
integer :: ierr
@@ -411,7 +411,7 @@ end subroutine petsc_matrix_get_node_indices
subroutine petsc_matrix_insert_block(mat, itri, m, n, val, iop)
use mesh_mod
implicit none
-!#include "finclude/petscvec.h"
+!#include
type(matrix_type) :: mat
integer, intent(in) :: itri, m, n, iop
@@ -447,7 +447,7 @@ subroutine identity_row(mat, irow)
use mesh_mod
use vector_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_matrix) :: mat
integer, intent(in) :: irow
@@ -464,7 +464,7 @@ end subroutine identity_row
subroutine set_row_vals(mat, irow, ncols, icols, vals)
use mesh_mod
use vector_mod
-#include "finclude/petscvec.h"
+!#include
type(petsc_matrix) :: mat
integer, intent(in) :: irow, ncols
@@ -485,7 +485,7 @@ end subroutine set_row_vals
subroutine petsc_matrix_write(m, file)
implicit none
-#include "finclude/petsc.h"
+!#include
type(matrix_type), intent(in) :: m
character(len=*) :: file
diff --git a/unstructured/petsc_vector.f90 b/unstructured/petsc_vector.f90
index 9df101234..532aad74b 100644
--- a/unstructured/petsc_vector.f90
+++ b/unstructured/petsc_vector.f90
@@ -2,7 +2,7 @@ module petsc_vector_mod
implicit none
-#include "finclude/petscvecdef.h"
+!#include
type :: petsc_vector
Vec :: vec
@@ -132,7 +132,7 @@ end function dof_index
subroutine petsc_vector_create(v,n)
use mesh_mod
implicit none
-#include "finclude/petsc.h"
+!#include
type(petsc_vector), intent(inout) :: v
integer, intent(in) :: n
@@ -183,7 +183,7 @@ end subroutine petsc_vector_destroy
subroutine petsc_vector_copy(vout,vin)
use mesh_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: vout
type(petsc_vector), intent(in) :: vin
@@ -223,7 +223,7 @@ end subroutine petsc_vector_copy
!======================================================================
subroutine petsc_vector_const_real(v,s)
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
real, intent(in) :: s
@@ -244,7 +244,7 @@ end subroutine petsc_vector_const_real
subroutine petsc_vector_add(vout,vin)
use mesh_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: vout
type(petsc_vector), intent(in) :: vin
@@ -284,7 +284,7 @@ end subroutine petsc_vector_add
!======================================================================
subroutine petsc_vector_multiply_real(v, s)
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
real, intent(in) :: s
integer :: ierr
@@ -319,7 +319,7 @@ end subroutine petsc_vector_multiply_complex
!======================================================================
subroutine petsc_vector_insert_real(v, i, s, iop)
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
real, intent(in) :: s
@@ -346,7 +346,7 @@ end subroutine petsc_vector_insert_real
#ifdef USECOMPLEX
subroutine petsc_vector_insert_complex(v, i, s, iop)
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
complex, intent(in) :: s
@@ -374,7 +374,7 @@ end subroutine petsc_vector_insert_complex
subroutine petsc_vector_sum_shared(v)
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
integer :: ierr
@@ -391,7 +391,7 @@ end subroutine petsc_vector_sum_shared
subroutine petsc_vector_finalize(v)
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
integer :: ierr
@@ -471,7 +471,7 @@ end subroutine petsc_vector_insert_block
logical function petsc_vector_is_nan(v)
implicit none
-#include "finclude/petsc.h"
+!#include
type(petsc_vector), intent(in) :: v
integer :: ierr
PetscScalar :: y(1)
@@ -489,7 +489,7 @@ subroutine petsc_vector_get_node_data_real(v, iplace, inode, data, rotate)
use element
use mesh_mod
implicit none
-#include "finclude/petsc.h"
+!#include
type(petsc_vector), intent(in) :: v
integer, intent(in) :: inode, iplace
@@ -546,7 +546,7 @@ subroutine petsc_vector_get_node_data_complex(v, iplace, inode, data, rotate)
use element
use mesh_mod
implicit none
-#include "finclude/petsc.h"
+!#include
type(petsc_vector), intent(in) :: v
integer, intent(in) :: inode, iplace
complex, intent(out), dimension(dofs_per_node) :: data
@@ -600,7 +600,7 @@ subroutine petsc_vector_set_node_data_real(v, iplace, inode, data, rotate)
use element
use mesh_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
integer, intent(in) :: inode, iplace
real, intent(in), dimension(dofs_per_node) :: data
@@ -649,7 +649,7 @@ subroutine petsc_vector_set_node_data_complex(v, iplace, inode, data, rotate)
use element
use mesh_mod
implicit none
-#include "finclude/petscvec.h"
+!#include
type(petsc_vector), intent(inout) :: v
integer, intent(in) :: inode, iplace
complex, intent(in), dimension(dofs_per_node) :: data
@@ -738,7 +738,7 @@ end function global_dof_id
subroutine petsc_vector_write(v, file)
implicit none
-#include "finclude/petsc.h"
+!#include
type(vector_type), intent(in) :: v
character(len=*) :: file
diff --git a/unstructured/read_ascii.f90 b/unstructured/read_ascii.f90
index 7b817343f..97929ac48 100644
--- a/unstructured/read_ascii.f90
+++ b/unstructured/read_ascii.f90
@@ -53,9 +53,9 @@ end subroutine get_token
! xrow = column to read (default = 1)
!======================================================================
subroutine read_ascii_column_real(filename, x, n, skip, icol, read_until)
+ use mpi
implicit none
- include 'mpif.h'
character(len=*), intent(in) :: filename
real, allocatable :: x(:)
diff --git a/unstructured/read_gyro.f90 b/unstructured/read_gyro.f90
index e16dee56b..cfe15066b 100644
--- a/unstructured/read_gyro.f90
+++ b/unstructured/read_gyro.f90
@@ -37,10 +37,10 @@ module gyro
contains
subroutine load_gyro(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
diff --git a/unstructured/read_neo.f90 b/unstructured/read_neo.f90
index 2771b6a9f..f2a94e40c 100644
--- a/unstructured/read_neo.f90
+++ b/unstructured/read_neo.f90
@@ -69,9 +69,9 @@ subroutine unload_neo()
end subroutine unload_neo
subroutine read_neo_expnorm(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: ifile=22
@@ -113,9 +113,9 @@ subroutine read_neo_expnorm(ierr)
end subroutine read_neo_expnorm
subroutine read_neo_grid(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: ifile=22
@@ -185,9 +185,9 @@ end subroutine read_neo_grid
subroutine read_neo_vel_fourier(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: ifile=23
diff --git a/unstructured/read_schaffer_field.f90 b/unstructured/read_schaffer_field.f90
index ba2c49f15..07ec53c5d 100644
--- a/unstructured/read_schaffer_field.f90
+++ b/unstructured/read_schaffer_field.f90
@@ -15,10 +15,10 @@ module read_schaffer_field
contains
subroutine load_schaffer_field(sf, filename, isamp, isamp_pol, ierr)
+ use mpi
use math
implicit none
- include 'mpif.h'
type(schaffer_field), intent(inout) :: sf
character(len=*), intent(in) :: filename
@@ -369,9 +369,9 @@ end subroutine get_external_field_ft
subroutine get_external_field(sf,r1,phi1,z1,br_out,bphi_out,bz_out,p_out, &
npts)
+ use mpi
implicit none
- include 'mpif.h'
type(schaffer_field), intent(in) :: sf
integer, intent(in) :: npts
diff --git a/unstructured/readaeqdsk.f90 b/unstructured/readaeqdsk.f90
index 01ec6405e..a2c52179d 100644
--- a/unstructured/readaeqdsk.f90
+++ b/unstructured/readaeqdsk.f90
@@ -71,11 +71,11 @@ module eqdsk_a
contains
subroutine load_eqdsk_a(filename)
+ use mpi
implicit none
#ifdef USEMPI
- include 'mpif.h'
#endif
character(len=*), intent(in) :: filename
diff --git a/unstructured/readgeqdsk.f90 b/unstructured/readgeqdsk.f90
index 9f69dcb8e..4b586ecf3 100644
--- a/unstructured/readgeqdsk.f90
+++ b/unstructured/readgeqdsk.f90
@@ -16,9 +16,9 @@ module eqdsk
contains
subroutine load_eqdsk(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
diff --git a/unstructured/region.f90 b/unstructured/region.f90
index af066ad13..b59eccbf5 100644
--- a/unstructured/region.f90
+++ b/unstructured/region.f90
@@ -66,9 +66,9 @@ subroutine destroy_plane(p)
end subroutine destroy_plane
subroutine create_region_from_file(r, filename, ierr)
+ use mpi
implicit none
- include 'mpif.h'
type(region_type), intent(inout) :: r
character(len=*), intent(in) :: filename
diff --git a/unstructured/restart_hdf5.f90 b/unstructured/restart_hdf5.f90
index ebc3b9579..176d612d5 100644
--- a/unstructured/restart_hdf5.f90
+++ b/unstructured/restart_hdf5.f90
@@ -9,6 +9,7 @@ module restart_hdf5
contains
subroutine rdrestart_hdf5()
+ use mpi
use basic
use hdf5_output
use hdf5
@@ -20,7 +21,6 @@ subroutine rdrestart_hdf5()
implicit none
- include 'mpif.h'
integer :: error
integer(HID_T) :: root_id, scalar_group_id, time_id, eq_time_id, pel_group_id, mesh_id
diff --git a/unstructured/rmp.f90 b/unstructured/rmp.f90
index d42d9887a..10ca2071a 100644
--- a/unstructured/rmp.f90
+++ b/unstructured/rmp.f90
@@ -409,6 +409,7 @@ end subroutine rmp_field
subroutine calculate_external_fields(ilin)
+ use mpi
use basic
use math
use mesh_mod
@@ -422,7 +423,6 @@ subroutine calculate_external_fields(ilin)
implicit none
- include 'mpif.h'
type(matrix_type) :: br_mat, bf_mat
type(vector_type) :: psi_vec, bz_vec, p_vec, bf_vec
@@ -849,6 +849,7 @@ subroutine boundary_rmp(rhs, mat)
end subroutine boundary_rmp
subroutine calculate_field_from_j(ilin)
+ use mpi
use basic
use math
use mesh_mod
@@ -861,7 +862,6 @@ subroutine calculate_field_from_j(ilin)
implicit none
- include 'mpif.h'
type(matrix_type) :: jx_mat
type(vector_type) :: psi_vec
diff --git a/unstructured/runaway_advection.f90 b/unstructured/runaway_advection.f90
index fcb3b9b5d..e7cff9850 100644
--- a/unstructured/runaway_advection.f90
+++ b/unstructured/runaway_advection.f90
@@ -98,9 +98,9 @@ module runaway_advection
! Note: any changes to the "particle" user-defined datatype must be reflected
! in the definitions of pnvars, pblklen, ptyps, and pdspls below.
subroutine define_mpi_particle(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: pnvars = 10
@@ -144,9 +144,9 @@ subroutine define_mpi_particle(ierr)
end subroutine define_mpi_particle
subroutine define_mpi_elfield(ierr)
+ use mpi
implicit none
- include 'mpif.h'
integer, intent(out) :: ierr
integer, parameter :: pnvars = 12
@@ -194,11 +194,11 @@ subroutine define_mpi_elfield(ierr)
end subroutine define_mpi_elfield
!---------------------------------------------------------------------------
subroutine runaway_advection_initialize
+ use mpi
use basic
use diagnostics
use auxiliary_fields
implicit none
- include 'mpif.h'
integer, parameter :: trunit = 120
real :: tstart, tend
@@ -248,13 +248,13 @@ real function tri_area(x1, z1, x2, z2, x3, z3)
end function tri_area
!---------------------------------------------------------------------------
subroutine init_particles(lrestart, ierr)
+ use mpi
use basic
use arrays
use m3dc1_nint
use gradshafranov
use read_ascii
implicit none
- include 'mpif.h'
logical, intent(in) :: lrestart
integer, intent(out) :: ierr
@@ -678,10 +678,10 @@ subroutine init_particles(lrestart, ierr)
end subroutine init_particles
subroutine advance_particles(tinc)
+ use mpi
use basic !For MPI variables
use omp_lib
implicit none
- include 'mpif.h'
real, intent(in) :: tinc !Time increment for particle advance
type(elfield), dimension(nneighbors + 1) :: elcoefs
@@ -1091,11 +1091,11 @@ subroutine fdot(x, v, w, dxdt, dvdt, dwdt, dEpdt, itri, ierr)
end subroutine fdot
!---------------------------------------------------------------------------
subroutine runaway_advection_step(pdt)
+ use mpi
use basic
use diagnostics
use auxiliary_fields
implicit none
- include 'mpif.h'
real, intent(in) :: pdt
@@ -1127,11 +1127,11 @@ subroutine runaway_advection_step(pdt)
end subroutine runaway_advection_step
!---------------------------------------------------------------------------
subroutine update_particle_pressure
+ use mpi
use basic
use arrays
use diagnostics
implicit none
- include 'mpif.h'
real :: tstart, tend
integer :: ierr
@@ -1540,11 +1540,11 @@ subroutine get_geom_terms(x, ielm, gh, ic2, ierr)
end subroutine get_geom_terms
!---------------------------------------------------------------------------
subroutine get_field_coefs(eq)
+ use mpi
use arrays
use basic
use auxiliary_fields
implicit none
- include 'mpif.h'
!type(elfield), intent(out) :: fh !Field handle
integer, intent(in) :: eq
diff --git a/unstructured/scorec_matrix.f90 b/unstructured/scorec_matrix.f90
index 53be5efdb..f6eb8f1e4 100644
--- a/unstructured/scorec_matrix.f90
+++ b/unstructured/scorec_matrix.f90
@@ -1,6 +1,7 @@
module scorec_matrix_mod
use element
+ use mpi
implicit none
@@ -187,16 +188,8 @@ subroutine scorec_matrix_create(mat, m, n, icomplex, lhs)
#ifndef M3DC1_TRILINOS
-#if PETSC_VERSION >= 38
use petsc
implicit none
-#elif PETSC_VERSION >= 36
- implicit none
-#include "petsc/finclude/petsc.h"
-#else
- implicit none
-#include "finclude/petsc.h"
-#endif
#endif
@@ -444,16 +437,8 @@ end subroutine scorec_matrix_insert_global_complex
subroutine scorec_matrix_solve(mat, v, ierr)
use vector_mod
-#if PETSC_VERSION >= 38
use petsc
implicit none
-#elif PETSC_VERSION >= 36
- implicit none
-#include "petsc/finclude/petsc.h"
-#else
- implicit none
-#include "finclude/petsc.h"
-#endif
type(scorec_matrix), intent(in) :: mat
type(vector_type), intent(inout) :: v
@@ -498,16 +483,8 @@ end subroutine scorec_matrix_solve
subroutine scorec_matrix_solve_with_guess(mat, v, x_guess, ierr)
use vector_mod
-#if PETSC_VERSION >= 38
use petsc
implicit none
-#elif PETSC_VERSION >= 36
- implicit none
-#include "petsc/finclude/petsc.h"
-#else
- implicit none
-#include "finclude/petsc.h"
-#endif
type(scorec_matrix), intent(in) :: mat
type(vector_type), intent(inout) :: v, x_guess
diff --git a/unstructured/scorec_mesh.f90 b/unstructured/scorec_mesh.f90
index 6bf808a4d..2d18d51e5 100644
--- a/unstructured/scorec_mesh.f90
+++ b/unstructured/scorec_mesh.f90
@@ -67,12 +67,12 @@ module scorec_mesh_mod
contains
subroutine load_mesh
+ use mpi
use math
use read_ascii
implicit none
integer :: myrank, maxrank, ier
- include 'mpif.h'
#ifdef USE3D
real :: angle, beta
integer :: i,procs_per_plane, full_group, plane_group
@@ -220,8 +220,8 @@ subroutine load_mesh
end subroutine load_mesh
subroutine update_nodes_owned
+ use mpi
integer :: numnodes, inode2, inode, ier, myrank, own_process
- include 'mpif.h'
call MPI_Comm_rank(MPI_COMM_WORLD,myrank,ier)
if(allocated(nodes_owned)) deallocate(nodes_owned)
allocate(nodes_owned(owned_nodes()))
@@ -262,10 +262,10 @@ subroutine print_node_data()
end subroutine print_node_data
subroutine get_global_dims
+ use mpi
implicit none
- include 'mpif.h'
integer :: nelms, error
diff --git a/unstructured/struct2vac.f90 b/unstructured/struct2vac.f90
index a0002542e..79ec8d26b 100644
--- a/unstructured/struct2vac.f90
+++ b/unstructured/struct2vac.f90
@@ -1,10 +1,10 @@
program struct2vac
+ use mpi
use mesh_mod
implicit none
- include 'mpif.h'
integer :: inode, ier, num_global_nodes, num_local_nodes
integer :: i, j, ilast, ifirst, numnodes
diff --git a/unstructured/test_region.f90 b/unstructured/test_region.f90
index 20f19d7a1..ba2ee84b3 100644
--- a/unstructured/test_region.f90
+++ b/unstructured/test_region.f90
@@ -1,10 +1,12 @@
Program test_region
+ use mpi
use region
implicit none
type(region_type) :: reg
integer :: ierr
+ character(len=256) :: boundary_file
integer, parameter :: n = 5
real, dimension(n) :: r, phi, z
@@ -18,13 +20,20 @@ Program test_region
r(5) = -2.; z(5) = -12.
phi = 0.
- call create_region_from_file(reg, "outer_boundary.pts", ierr)
+ call MPI_Init(ierr)
+
+ boundary_file = "unstructured/templates/ITER/rw2_adapt/outer_boundary.pts"
+ if(command_argument_count().ge.1) call get_command_argument(1, boundary_file)
+
+ call create_region_from_file(reg, trim(boundary_file), ierr)
do i=1, n
in = point_in_region(reg, r(i), phi(i), z(i))
- write(*,'("Is (", G, ", ", G," in region? ", L)') r(i), z(i), in
+ write(*,'("Is (", G0, ", ", G0, " in region? ", L1)') r(i), z(i), in
end do
call destroy_region(reg)
+ call MPI_Finalize(ierr)
+
end Program test_region
diff --git a/unstructured/time_step.f90 b/unstructured/time_step.f90
index 470eba0b1..3be287f16 100644
--- a/unstructured/time_step.f90
+++ b/unstructured/time_step.f90
@@ -326,13 +326,13 @@ end subroutine scaleback
subroutine variable_timestep
+ use mpi
use basic
use arrays
use diagnostics
implicit none
- include 'mpif.h'
integer :: ierr
#ifdef USE3D
diff --git a/unstructured/transport.f90 b/unstructured/transport.f90
index 9d4da8503..4b0ed5465 100644
--- a/unstructured/transport.f90
+++ b/unstructured/transport.f90
@@ -1577,6 +1577,7 @@ end function bs_func
! define_transport_coefficients
! =============================
subroutine define_transport_coefficients()
+ use mpi
use basic
use arrays
@@ -1590,7 +1591,6 @@ subroutine define_transport_coefficients()
implicit none
- include 'mpif.h'
integer :: itri, izone, izone_index
integer :: numelms, def_fields,ier
diff --git a/unstructured/vector_mod.F90 b/unstructured/vector_mod.F90
new file mode 100644
index 000000000..8aa9ec07b
--- /dev/null
+++ b/unstructured/vector_mod.F90
@@ -0,0 +1,14 @@
+! vector_mod.F90 — thin wrapper that selects the vector backend at compile time.
+! CMake defines exactly one of the following:
+! M3DC1_VECTOR_IS_SCOREC → use scorec_vector_mod
+! M3DC1_VECTOR_IS_PETSC → use petsc_vector_mod
+
+module vector_mod
+#ifdef M3DC1_VECTOR_IS_SCOREC
+ use scorec_vector_mod
+#elif defined(M3DC1_VECTOR_IS_PETSC)
+ use petsc_vector_mod
+#else
+ use scorec_vector_mod
+#endif
+end module vector_mod