CMake build system

.travis.yml

@@ -6,6 +6,7 @@ addons:
     packages:
       - libtool-bin
       - libmpich-dev
+      - mpich
 
 install:
   # Install ROSS
@@ -46,7 +47,22 @@ install:
     export PATH=$PATH:${TRAVIS_BUILD_DIR}/ci-deps/OTF2/bin
 
 script:
-  - cd tracer
+  - pushd tracer
   - make all PREFIX=${TRAVIS_BUILD_DIR}/install-test CXX=mpicc CXX=mpicxx ROSS_DIR="${TRAVIS_BUILD_DIR}/ci-deps/ROSS" CODES_DIR="${TRAVIS_BUILD_DIR}/ci-deps/CODES" SELECT_TRACE="-DTRACER_OTF_TRACES=1"
+  - make install PREFIX=${TRAVIS_BUILD_DIR}/install-test CXX=mpicc CXX=mpicxx ROSS_DIR="${TRAVIS_BUILD_DIR}/ci-deps/ROSS" CODES_DIR="${TRAVIS_BUILD_DIR}/ci-deps/CODES" SELECT_TRACE="-DTRACER_OTF_TRACES=1"
   - cd ../utils
   - make
+  - popd
+  - mkdir ${TRAVIS_BUILD_DIR}/tests/output
+  - ${TRAVIS_BUILD_DIR}/tests/bin/run-example-test-with-conf.sh "stencil4d-otf" "dfly"
+  - ${TRAVIS_BUILD_DIR}/tests/bin/run-example-test-with-conf.sh "stencil4d-otf" "fattree"
+  - |
+    pushd examples/stencil4d-otf
+    mpirun -np 2 ${TRAVIS_BUILD_DIR}/install-test/bin/traceR --lp-io-dir=${TRAVIS_BUILD_DIR}/tests/output/test-output-hypre --sync=3 -- ../conf/hypreX_expressMesh.conf tracer_config | sed -n '/START PARALLEL OPTIMISTIC SIMULATION/,/END SIMULATION/p' | tail -n +2 | head -n -3 > std.out
+    rv=$?
+    cat std.out
+    mv std.out ${TRAVIS_BUILD_DIR}/tests/output/test-output-hypre/std.out
+    popd
+    if [[ $rv == 0 ]]; then true; else false; fi
+  - ${TRAVIS_BUILD_DIR}/tests/bin/check-files-match.sh ${TRAVIS_BUILD_DIR}/tests/regression/test-output-hypre
+  - ${TRAVIS_BUILD_DIR}/tests/bin/run-example-test-with-conf.sh "stencil4d-otf" "torus"

CMakeLists.txt

@@ -0,0 +1,42 @@
+cmake_minimum_required(VERSION 3.6)
+
+project(tracer VERSION 0.0.0)
+
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
+
+# Options
+option(TRACER_ENABLE_OTF2 "Enables OTF2 trace support" ON)
+option(TRACER_ENABLE_BIGSIM "Enables BigSim trace support" OFF)
+option(TRACER_ENABLE_EXAMPLES "Enables building the examples" OFF)
+option(TRACER_ENABLE_DOCS "Enables building the docs" OFF)
+option(TRACER_ENABLE_MAIN_EXE "Enables building the main TraceR executable" ON)
+
+if(TRACER_ENABLE_OTF2 AND TRACER_ENABLE_BIGSIM)
+    message(FATAL_ERROR "OTF2 and BigSim support cannot both be enabled at this time")
+elseif(NOT TRACER_ENABLE_OTF2 AND NOT TRACER_ENABLE_BIGSIM)
+    message(FATAL_ERROR "One of OTF2 or BigSim support must be enabled (but not both)")
+endif()
+
+# Setting for IDEs
+set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+
+# Main tracer executable
+if(TRACER_ENABLE_MAIN_EXE)
+    add_subdirectory(tracer)
+
+    # Enable tests
+    include(CTest)
+    add_subdirectory(tests)
+endif()
+
+# Examples
+if(TRACER_ENABLE_EXAMPLES)
+    message(SEND_ERROR "Support for building examples with CMake is not currently available")
+    #add_subdirectory(examples)
+endif()
+
+# Docs
+if(TRACER_ENABLE_DOCS)
+    add_subdirectory(docs)
+endif()
+

cmake/FindBigSim.cmake

@@ -0,0 +1,42 @@
+# Find the Charm++ include folder
+find_path(CHARM_INCLUDE_DIR charm.h
+    HINTS $ENV{CHARMPATH} ${CHARMPATH}
+    PATH_SUFFIXES include
+)
+if(NOT CHARM_INCLUDE_DIR)
+    message(FATAL_ERROR "charm include directory was not found, set CHARMPATH to the install prefix for Charm++")
+endif()
+
+# Find Charm++ libraries used
+find_library(CHARM_CONV_BIGSIM_LOGS_LIBRARY NAMES conv-bigsim-logs
+    HINTS $ENV{CHARMPATH} ${CHARMPATH}
+    PATH_SUFFIXES lib
+)
+find_library(CHARM_BLUE_STANDALONE_LIBRARY NAMES blue-standalone
+    HINTS $ENV{CHARMPATH} ${CHARMPATH}
+    PATH_SUFFIXES lib
+)
+find_library(CHARM_CONV_UTIL_LIBRARY NAMES conv-util
+    HINTS $ENV{CHARMPATH} ${CHARMPATH}
+    PATH_SUFFIXES lib
+)
+list(APPEND CHARM_LIBRARIES "${CHARM_CONV_BIGSIM_LOGS_LIBRARY}")
+list(APPEND CHARM_LIBRARIES "${CHARM_BLUE_STANDALONE_LIBRARY}")
+list(APPEND CHARM_LIBRARIES "${CHARM_CONV_UTIL_LIBRARY}")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Charm
+    REQUIRED_VARS
+        CHARM_INCLUDE_DIR
+        CHARM_LIBRARIES
+        CHARM_CONV_BIGSIM_LOGS_LIBRARY
+        CHARM_BLUE_STANDALONE_LIBRARY
+        CHARM_CONV_UTIL_LIBRARY
+)
+
+if(CHARM_FOUND AND NOT TARGET CHARM::BigSim)
+    add_library(CHARM::BigSim INTERFACE IMPORTED)
+    # Target flags needed
+    target_include_directories(CHARM::BigSim INTERFACE ${CHARM_INCLUDE_DIR})
+    target_link_libraries(CHARM::BigSim INTERFACE ${CHARM_CONV_BIGSIM_LOGS_LIBRARY} ${CHARM_BLUE_STANDALONE_LIBRARY} ${CHARM_CONV_UTIL_LIBRARY})
+endif()

cmake/FindOTF2.cmake

@@ -0,0 +1,36 @@
+# Find the otf2-config program and run it to get paths and flags for OTF2
+find_program(OTF2_CONFIG_EXECUTABLE NAMES otf2-config
+    HINTS $ENV{OTF2_DIR} $ENV{SCOREP_DIR} ${SCOREP_DIR} ${OTF2_DIR}
+    PATH_SUFFIXES bin
+)
+
+if(NOT OTF2_CONFIG_EXECUTABLE)
+    message(FATAL_ERROR "otf2-config was not found, set OTF2_DIR to the install prefix for otf2/score-p")
+endif()
+
+# Get compile/link flags for otf2
+execute_process(COMMAND ${OTF2_CONFIG_EXECUTABLE} --libs
+    OUTPUT_VARIABLE OTF2_LIBRARIES
+)
+execute_process(COMMAND ${OTF2_CONFIG_EXECUTABLE} --cflags
+    OUTPUT_VARIABLE OTF2_COMPILE_FLAGS
+)
+execute_process(COMMAND ${OTF2_CONFIG_EXECUTABLE} --ldflags
+    OUTPUT_VARIABLE OTF2_LINK_FLAGS
+)
+
+# Leading/trailing whitespace is an error, remove it
+string(STRIP ${OTF2_LIBRARIES} OTF2_LIBRARIES)
+string(STRIP ${OTF2_COMPILE_FLAGS} OTF2_COMPILE_FLAGS)
+string(STRIP ${OTF2_LINK_FLAGS} OTF2_LINK_FLAGS)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(OTF2
+    REQUIRED_VARS OTF2_CONFIG_EXECUTABLE OTF2_LIBRARIES OTF2_COMPILE_FLAGS OTF2_LINK_FLAGS
+)
+
+if(OTF2_FOUND AND NOT TARGET OTF2::OTF2)
+    add_library(OTF2::OTF2 INTERFACE IMPORTED)
+    target_compile_options(OTF2::OTF2 INTERFACE ${OTF2_COMPILE_FLAGS})
+    target_link_libraries(OTF2::OTF2 INTERFACE ${OTF2_LIBRARIES} ${OTF2_LINK_FLAGS})
+endif()

docs/CMakeLists.txt

@@ -0,0 +1,16 @@
+find_program(SPHINX_EXECUTABLE NAMES sphinx-build
+    HINTS $ENV{SPHINX_DIR} ${SPHINX_DIR}
+    PATH_SUFFIXES bin
+)
+
+if(NOT SPHINX_EXECUTABLE)
+    message(FATAL_ERROR "sphinx-build was not found, set SPHINX_DIR to the install prefix for sphinx")
+endif()
+
+add_custom_target(tracer_docs ALL
+    ${SPHINX_EXECUTABLE}
+        -M
+        "${CMAKE_CURRENT_SOURCE_DIR}"
+        "${CMAKE_CURRENT_BINARY_DIR}"
+    COMMENT "Building TraceR documentation with Sphinx"
+)

examples/CMakeLists.txt

@@ -0,0 +1,7 @@
+if(TRACER_ENABLE_OTF2)
+    add_subdirectory(stencil4d-otf)
+endif()
+
+if(TRACER_ENABLE_BIGSIM)
+    add_subdirectory(jacobi2d-bigsim)
+endif()

examples/jacobi2d-bigsim/CMakeLists.txt

@@ -0,0 +1,17 @@
+# This likely won't work since it is trying to switch the compiler,
+# which is an area that would be useful to have Charm support CMake
+# similar to the support Qt support in CMake where the wrapper/generator
+# programs are run automatically.
+
+find_program(AMPICC_EXECUTABLE NAMES ampiCC
+    HINTS $ENV{CHARMPATH} ${CHARMPATH}
+    PATH_SUFFIXES bin
+)
+if(NOT AMPICC_EXECUTABLE)
+    message(FATAL_ERROR "ampiCC was not found, set CHARMPATH to the install prefix for Charm++")
+endif()
+
+set(CMAKE_CXX_COMPILER ${AMPICC_EXECUTABLE})
+set(CMAKE_C_COMPILER ${AMPICC_EXECUTABLE})
+
+add_executable(jacobi2d jacobi2d.C)

examples/stencil4d-otf/CMakeLists.txt

@@ -0,0 +1,66 @@
+# THIS DOES NOT WORK AT THE MOMENT
+# Score-P has a compiler wrapper that plays weird games with linking/compiling flags
+# There are a pair of undefined symbols, scorep_subsystems and scorep_number_of_subsystems
+# that are located in the libscorep_measurement.so and .a libraries (as shown by running nm on them).
+# Somehow with the make file that doesn't have a separate linking stage,  these undefined symbols suddenly
+# become defined in the compiled executable. After hours digging into it, I still haven't figured out
+# what is going on; this is an issue that probably will require support from the Score-P developers,
+# ideally in the form of a CMake package with macros for adding targets that are compiled using Score-P.
+
+find_package(MPI REQUIRED)
+
+# Find the scorep compiler wrapper
+find_program(SCOREP_EXECUTABLE NAMES scorep
+    HINTS $ENV{SCOREP_DIR} ${SCOREP_DIR}
+    PATH_SUFFIXES bin
+)
+
+if(NOT SCOREP_EXECUTABLE)
+    message(FATAL_ERROR "scorep was not found, set SCOREP_DIR to the install prefix for score-p")
+endif()
+
+# Find scorep-config
+find_program(SCOREP_CONFIG_EXECUTABLE NAMES scorep-config
+    HINTS $ENV{SCOREP_DIR} ${SCOREP_DIR}
+    PATH_SUFFIXES bin
+)
+
+if(NOT SCOREP_CONFIG_EXECUTABLE)
+    message(FATAL_ERROR "scorep-config was not found, set SCOREP_DIR to the install prefix for score-p")
+endif()
+
+execute_process(COMMAND ${SCOREP_CONFIG_EXECUTABLE} --user --cppflags
+    OUTPUT_VARIABLE SCOREP_CXX_FLAGS
+)
+execute_process(COMMAND ${SCOREP_CONFIG_EXECUTABLE} --user --libs
+    OUTPUT_VARIABLE SCOREP_LIBRARIES
+)
+execute_process(COMMAND ${SCOREP_CONFIG_EXECUTABLE} --user --ldflags
+    OUTPUT_VARIABLE SCOREP_LDFLAGS
+)
+
+# Leading/trailing whitespace is an error, remove it
+string(STRIP ${SCOREP_CXX_FLAGS} SCOREP_CXX_FLAGS)
+string(STRIP ${SCOREP_LIBRARIES} SCOREP_LIBRARIES)
+# Remove "-tcollect"; unrecognized compiler option
+string(REPLACE "-tcollect" "" SCOREP_LDFLAGS ${SCOREP_LDFLAGS})
+string(STRIP ${SCOREP_LDFLAGS} SCOREP_LDFLAGS)
+
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SCOREP_LDFLAGS}")
+
+# Compile stencil4d with the scorep wrapper
+list(APPEND SCOREP_LAUNCHER "${SCOREP_EXECUTABLE}" "--user" "--nocompiler" "--noopenmp" "--nopomp" "--nocuda" "--noopenacc" "--noopencl" "--nomemory")
+set(CMAKE_C_COMPILER_LAUNCHER ${SCOREP_LAUNCHER})
+set(CMAKE_CXX_COMPILER_LAUNCHER ${SCOREP_LAUNCHER})
+
+# Create the stencil4d target
+add_executable(stencil4d stencil4d.C)
+target_compile_options(stencil4d PRIVATE ${SCOREP_CXX_FLAGS})
+target_link_libraries(stencil4d ${SCOREP_LIBRARIES})
+target_compile_definitions(stencil4d PRIVATE WRITE_OTF2_TRACE=1)
+
+# With CMake 3.9, this is simplified to just target_link_libraries(traceR MPI::MPI_C)
+target_include_directories(stencil4d SYSTEM PRIVATE ${MPI_C_INCLUDE_PATH})
+target_link_libraries(stencil4d ${MPI_C_LIBRARIES})
+target_compile_options(stencil4d PRIVATE ${MPI_C_COMPILE_FLAGS})
+target_link_libraries(stencil4d ${MPI_C_LINK_FLAGS})

tests/CMakeLists.txt

@@ -0,0 +1,18 @@
+set(TEST_OUT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/output")
+set(TEST_RUN_SCRIPT "${CMAKE_CURRENT_SOURCE_DIR}/bin/run-example-test-with-conf.sh")
+
+add_test(NAME stencil4d-otf-dfly
+    COMMAND ${TEST_RUN_SCRIPT} "stencil4d-otf" "dfly"
+)
+
+add_test(NAME stencil4d-otf-fattree
+    COMMAND ${TEST_RUN_SCRIPT} "stencil4d-otf" "fattree"
+)
+
+add_test(NAME stencil4d-otf-hypre
+    COMMAND ${TEST_RUN_SCRIPT} "stencil4d-otf" "hypreX_expressMesh"
+)
+
+add_test(NAME stencil4d-otf-torus
+    COMMAND ${TEST_RUN_SCRIPT} "stencil4d-otf" "torus"
+)