diff --git a/.github/workflows/master-test-workflow.yml b/.github/workflows/master-test-workflow.yml index 0709573..03473f7 100644 --- a/.github/workflows/master-test-workflow.yml +++ b/.github/workflows/master-test-workflow.yml @@ -42,7 +42,7 @@ jobs: - name: Dry-run compilation shell: bash - run: source /home/hicr/.bashrc && meson setup --wipe build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,lpf,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=false && meson compile -C build + run: source /home/hicr/.bashrc && meson setup --wipe build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,lpf,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=true && meson compile -C build - name: Running tests and creating coverage report shell: bash @@ -51,7 +51,7 @@ jobs: run: | echo "Running Tests..." source /home/hicr/.bashrc - meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=false + meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=true meson compile -C build meson test -C build echo "Creating coverage report..." diff --git a/.github/workflows/pr-development-workflow.yml b/.github/workflows/pr-development-workflow.yml index 34d176b..991db8a 100644 --- a/.github/workflows/pr-development-workflow.yml +++ b/.github/workflows/pr-development-workflow.yml @@ -231,7 +231,7 @@ jobs: docker run --name taskr --shm-size=1024M --privileged -v $PWD:/home/hicr/taskr -w /home/hicr/taskr -td ${{ env.DOCKERIMAGE }}:${{ inputs.arch }}-latest bash - name: Dry-run compilation - run: docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,lpf,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=false && meson compile -C build" + run: docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,lpf,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=true && meson compile -C build" - name: Compile run: docker exec -u hicr taskr bash -c "meson compile -C build" @@ -239,7 +239,7 @@ jobs: - name: Running tests and creating coverage report run: | echo "Running Tests..." - docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=false && meson compile -C build && meson test -C build" + docker exec -u hicr taskr bash -c "meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=true && meson compile -C build && meson test -C build" echo "Creating coverage report..." docker exec -u hicr taskr bash -c "ninja -C build coverage" docker stop taskr @@ -271,7 +271,7 @@ jobs: submodules: 'true' - name: Dry-run compilation - run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,lpf,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=false && meson compile -C build + run: source /home/hicr/.bashrc && meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,lpf,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=true && meson compile -C build - name: Running tests and creating coverage report shell: bash @@ -279,7 +279,7 @@ jobs: LD_PRELOAD: /usr/local/lib/libnosv.so run: | source /home/hicr/.bashrc - meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=false + meson setup build -Dbuildtype=debug -Db_coverage=true -DdistributedEngines=mpi,none -DbuildTests=true -DbuildExamples=true -DcompileWarningsAsErrors=true -DexecutionStateType=nosv,boost -DprocessingUnitType=nosv,pthreads -DbuildPyTaskR=true -DbuildInstrumentation=true meson compile -C build meson test -C build echo "Creating coverage report..." diff --git a/.gitignore b/.gitignore index 9d449eb..6ed4039 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ build atlas_*.sh **/matrix/ **/__pycache__/ -**/ovni/ \ No newline at end of file +**/ovni/ +**/tracr/ diff --git a/README.md b/README.md index f64891b..2b1f1e3 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,25 @@ ___________ __ __________ TaskR is a lightweight dependency-driven tasking runtime system based entirely on function calls to the [HiCR API](https://github.com/Algebraic-Programming/HiCR) +## Instrumentation (TraCR) + +TaskR can trace its runtime behavior (task execution, polling, worker suspension) with [TraCR](https://github.com/Algebraic-Programming/TraCR): + +```bash +meson setup build -DbuildInstrumentation=true ... +``` + +An instrumented run writes a `tracr/` folder into the working directory. Convert it with the `tracr_process` tool (built alongside TaskR) into the format of your choice: + +```bash +# Perfetto (open in https://ui.perfetto.dev), Paraver, or a terminal dump +./build/extern/tracr/postprocessing/tracr_process tracr/ perfetto +``` + +Each worker appears as its own named track with `executing` (annotated with the task id), `polling`, `suspended`, `resumed` and `finished` states. + +Note: tracing requires workers pinned to their own OS thread, i.e. the `pthreads` processing unit backend. With other backends (e.g. nOS-V), instrumentation automatically stays off. + ## License Copyright 2025 Huawei Technologies Co., Ltd. diff --git a/examples/matmul/python/matmul.cpp b/examples/matmul/python/matmul.cpp index 2ef0c04..8ffc643 100644 --- a/examples/matmul/python/matmul.cpp +++ b/examples/matmul/python/matmul.cpp @@ -19,10 +19,6 @@ #include -#ifdef ENABLE_INSTRUMENTATION - #include -#endif - #define mytype float /** @@ -30,10 +26,6 @@ */ void matmul(taskr::Task *) { -#ifdef ENABLE_INSTRUMENTATION - INSTRUMENTATION_VMARKER_SET(MARK_COLOR_RED); -#endif - const size_t N = 200; // Allocate memory @@ -64,10 +56,6 @@ void matmul(taskr::Task *) free((mytype *)A); free((mytype *)B); free((mytype *)C); - -#ifdef ENABLE_INSTRUMENTATION - INSTRUMENTATION_VMARKER_RESET(); -#endif } PYBIND11_MODULE(cpp_matmul, m) diff --git a/extern/HiCR b/extern/HiCR index 6eb59ea..b855a94 160000 --- a/extern/HiCR +++ b/extern/HiCR @@ -1 +1 @@ -Subproject commit 6eb59ea9e63461c847ec06a2dc7819ba41f509fc +Subproject commit b855a94e2dfc2f232dc52548cb9f6737d4a974dc diff --git a/extern/tracr b/extern/tracr index d37ea2a..ccf28f0 160000 --- a/extern/tracr +++ b/extern/tracr @@ -1 +1 @@ -Subproject commit d37ea2af0c4505a99c59bccb416e8c119404f290 +Subproject commit ccf28f0746731f456d29ee4fee01ecca444ec1bc diff --git a/include/taskr/runtime.hpp b/include/taskr/runtime.hpp index 2186465..b9bebcc 100644 --- a/include/taskr/runtime.hpp +++ b/include/taskr/runtime.hpp @@ -34,8 +34,8 @@ #include #include -#ifdef ENABLE_INSTRUMENTATION - #include +#ifdef ENABLE_TRACR + #include #endif #include "queue.hpp" @@ -48,39 +48,39 @@ namespace taskr { /** - * Thread indices for the TraCR thread markers + * TraCR eventIds of the marker types used by the TaskR runtime */ struct ThreadIndices { /** - * thread idx exec_task + * eventId of the "executing" marker */ - size_t exec_task; + uint16_t exec_task; /** - * thread idx exec_serv + * eventId of the "executing a service" marker */ - size_t exec_serv; + uint16_t exec_serv; /** - * thread idx polling + * eventId of the "polling" marker */ - size_t polling; + uint16_t polling; /** - * thread idx suspending + * eventId of the "suspended" marker */ - size_t suspending; + uint16_t suspending; /** - * thread idx resuming + * eventId of the "resumed" marker */ - size_t resuming; + uint16_t resuming; /** - * thread idx finished + * eventId of the "finished" marker */ - size_t finished; + uint16_t finished; }; /** @@ -129,23 +129,26 @@ class Runtime _workerComputeManager(workerComputeManager), _computeResources(computeResources) { -#ifdef ENABLE_INSTRUMENTATION - // This is to check if ovni has been already initialized by nOS-V - bool external_init_ = (dynamic_cast(_workerComputeManager) == nullptr) ? true : false; - - // TraCR start tracing - INSTRUMENTATION_START(external_init_); - - // TraCR initialize marker type - INSTRUMENTATION_THREAD_MARK_INIT(0); - - // TraCR marker types with the given string messages - thread_idx.exec_task = INSTRUMENTATION_THREAD_MARK_ADD(MARK_COLOR_GREEN, "executing"); - thread_idx.exec_serv = INSTRUMENTATION_THREAD_MARK_ADD(MARK_COLOR_CYAN, "executing a service"); - thread_idx.polling = INSTRUMENTATION_THREAD_MARK_ADD(MARK_COLOR_NAVY, "polling"); - thread_idx.suspending = INSTRUMENTATION_THREAD_MARK_ADD(MARK_COLOR_LIGHT_GRAY, "suspended"); - thread_idx.resuming = INSTRUMENTATION_THREAD_MARK_ADD(MARK_COLOR_LIGHT_GREEN, "resumed"); - thread_idx.finished = INSTRUMENTATION_THREAD_MARK_ADD(MARK_COLOR_YELLOW, "finished"); +#ifdef ENABLE_TRACR + // TraCR traces each OS thread separately, so it requires workers pinned to their own + // OS thread (pthreads backend). With other worker backends (e.g. nOS-V), workers may + // migrate between OS threads, hence instrumentation stays disabled for them. + // Only the first active TaskR runtime in the process is instrumented. + _instrumentationActive = (dynamic_cast(_workerComputeManager) != nullptr) && (INSTRUMENTATION_PROC_EXISTS() == false); + + if (_instrumentationActive) + { + // TraCR start tracing (this also registers the calling (main) thread) + INSTRUMENTATION_START(); + + // TraCR marker types with the given string messages + thread_idx.exec_task = INSTRUMENTATION_MARK_W_COLOR_ADD("executing", MARK_COLOR_GREEN); + thread_idx.exec_serv = INSTRUMENTATION_MARK_W_COLOR_ADD("executing a service", MARK_COLOR_CYAN); + thread_idx.polling = INSTRUMENTATION_MARK_W_COLOR_ADD("polling", MARK_COLOR_NAVY); + thread_idx.suspending = INSTRUMENTATION_MARK_W_COLOR_ADD("suspended", MARK_COLOR_LIGHT_GRAY); + thread_idx.resuming = INSTRUMENTATION_MARK_W_COLOR_ADD("resumed", MARK_COLOR_LIGHT_GREEN); + thread_idx.finished = INSTRUMENTATION_MARK_W_COLOR_ADD("finished", MARK_COLOR_YELLOW); + } #endif // Creating internal tasks @@ -383,9 +386,13 @@ class Runtime // Set state back to initialized _state = state_t::initialized; -#ifdef ENABLE_INSTRUMENTATION - // TraCR set trace of the main thread being finished - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.finished); +#ifdef ENABLE_TRACR + if (_instrumentationActive) + { + // TraCR set trace of the main thread being finished (and close its channel) + INSTRUMENTATION_MARK_SET(MAIN_CHANNEL_ID, thread_idx.finished, UINT32_MAX); + INSTRUMENTATION_MARK_RESET(MAIN_CHANNEL_ID); + } #endif } @@ -399,17 +406,29 @@ class Runtime if (_state == state_t::uninitialized) HICR_THROW_LOGIC("Trying to finalize TaskR, but it is currently not initialized"); if (_state == state_t::running) HICR_THROW_LOGIC("Trying to finalize TaskR, but it is currently running. You need to run 'await' first to make sure it has stopped."); +#ifdef ENABLE_TRACR + if (_instrumentationActive) + { + // Human-readable channel names for the visualization (channel = worker) + nlohmann::json channelNames = nlohmann::json::array(); + channelNames.push_back("main"); + for (size_t i = 0; i < _serviceWorkers.size(); i++) channelNames.push_back("service_worker_" + std::to_string(i)); + for (size_t i = 0; i < _taskWorkers.size(); i++) channelNames.push_back("task_worker_" + std::to_string(i)); + INSTRUMENTATION_ADD_CHANNEL_NAMES(channelNames); + + // TraCR stop tracing (flushes the traces and writes the metadata file) + INSTRUMENTATION_END(); + + _instrumentationActive = false; + } +#endif + // Clearing created workers _serviceWorkers.clear(); _taskWorkers.clear(); // Setting state back to uninitialized _state = state_t::uninitialized; - -#ifdef ENABLE_INSTRUMENTATION - // TraCR stop tracing - INSTRUMENTATION_END(); -#endif } /** @@ -473,6 +492,22 @@ class Runtime private: +#ifdef ENABLE_TRACR + /** + * Records a worker state transition with TraCR. Repeats of the current state are + * skipped so that the tight polling loop does not flood the trace buffer. + * + * @param[in] eventId The marker type of the state to record + * @param[in] extraId Optional extra information (e.g. the task id); forces the record + */ + __INLINE__ void instrumentWorkerState(const uint16_t eventId, const uint32_t extraId = UINT32_MAX) + { + if (extraId == UINT32_MAX && _tl_lastEventId == eventId) return; + _tl_lastEventId = eventId; + INSTRUMENTATION_MARK_SET(_tl_channelId, eventId, extraId); + } +#endif + __INLINE__ void tryRunService(Service *const service) { // Checking if service is enabled @@ -482,8 +517,8 @@ class Runtime if (service->isActive()) { // TraCR set trace of thread executing a service -#ifdef ENABLE_INSTRUMENTATION - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.exec_serv); +#ifdef ENABLE_TRACR + if (_instrumentationActive) instrumentWorkerState(thread_idx.exec_serv); #endif // Now run service @@ -494,9 +529,10 @@ class Runtime __INLINE__ taskr::Task *serviceWorkerLoop(const workerId_t serviceWorkerId) { -#ifdef ENABLE_INSTRUMENTATION - // TraCR set trace of thread polling - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.polling); +#ifdef ENABLE_TRACR + // Remember this worker's TraCR channel and set trace of thread polling + _tl_channelId = static_cast(1 + serviceWorkerId); + if (_instrumentationActive) instrumentWorkerState(thread_idx.polling); #endif // Getting worker pointer @@ -527,9 +563,10 @@ class Runtime __INLINE__ taskr::Task *taskWorkerLoop(const workerId_t taskWorkerId) { -#ifdef ENABLE_INSTRUMENTATION - // TraCR set trace of thread polling - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.polling); +#ifdef ENABLE_TRACR + // Remember this worker's TraCR channel and set trace of thread polling + _tl_channelId = static_cast(1 + _serviceWorkerCount + taskWorkerId); + if (_instrumentationActive) instrumentWorkerState(thread_idx.polling); #endif // The worker is once again active @@ -673,6 +710,11 @@ class Runtime // Getting TaskR task pointer auto taskrTask = (taskr::Task *)task; +#ifdef ENABLE_TRACR + // TraCR set trace of thread executing a task (annotated with the task id) + if (_instrumentationActive) instrumentWorkerState(thread_idx.exec_task, static_cast(taskrTask->getTaskId())); +#endif + // If defined, trigger user-defined event _taskCallbackMap.trigger(taskrTask, HiCR::tasking::Task::callback_t::onTaskExecute); } @@ -682,6 +724,11 @@ class Runtime // Getting TaskR task pointer auto taskrTask = (taskr::Task *)task; +#ifdef ENABLE_TRACR + // TraCR set trace of thread going back to polling + if (_instrumentationActive) instrumentWorkerState(thread_idx.polling); +#endif + // Setting task as finished task setFinishedTask(taskrTask); @@ -697,6 +744,11 @@ class Runtime // Getting TaskR task pointer auto taskrTask = (taskr::Task *)task; +#ifdef ENABLE_TRACR + // TraCR set trace of thread going back to polling + if (_instrumentationActive) instrumentWorkerState(thread_idx.polling); +#endif + // If defined, trigger user-defined event this->_taskCallbackMap.trigger(taskrTask, HiCR::tasking::Task::callback_t::onTaskSuspend); } @@ -712,9 +764,13 @@ class Runtime __INLINE__ void onWorkerStartCallback(HiCR::tasking::Worker *const worker) { -#ifdef ENABLE_INSTRUMENTATION - // TraCR initialize the thread - INSTRUMENTATION_THREAD_INIT(); +#ifdef ENABLE_TRACR + // TraCR initialize the worker thread + if (_instrumentationActive) + { + INSTRUMENTATION_THREAD_INIT(); + _tl_lastEventId = UINT16_MAX; + } #endif // Getting TaskR worker pointer @@ -729,9 +785,9 @@ class Runtime // Getting TaskR worker pointer auto taskrWorker = (taskr::Worker *)worker; -#ifdef ENABLE_INSTRUMENTATION +#ifdef ENABLE_TRACR // TraCR set trace of thread suspended - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.suspending); + if (_instrumentationActive) instrumentWorkerState(thread_idx.suspending); #endif // If defined, trigger user-defined event @@ -743,9 +799,9 @@ class Runtime // Getting TaskR worker pointer auto taskrWorker = (taskr::Worker *)worker; -#ifdef ENABLE_INSTRUMENTATION +#ifdef ENABLE_TRACR // TraCR set trace of thread resumed - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.resuming); + if (_instrumentationActive) instrumentWorkerState(thread_idx.resuming); #endif // If defined, trigger user-defined event @@ -757,12 +813,16 @@ class Runtime // Getting TaskR worker pointer auto taskrWorker = (taskr::Worker *)worker; -#ifdef ENABLE_INSTRUMENTATION - // Set the marker of this thread to be finished - INSTRUMENTATION_THREAD_MARK_SET(thread_idx.finished); +#ifdef ENABLE_TRACR + if (_instrumentationActive) + { + // Set the marker of this thread to be finished (and close its channel) + instrumentWorkerState(thread_idx.finished); + INSTRUMENTATION_MARK_RESET(_tl_channelId); - // TraCR end thread (only if backend is not nOS-V) - INSTRUMENTATION_THREAD_END(); + // TraCR flush and finalize this worker thread + INSTRUMENTATION_THREAD_FINALIZE(); + } #endif // If defined, trigger user-defined event @@ -900,9 +960,29 @@ class Runtime bool _finishOnLastTask; /** - * TraCR thread indices + * TraCR eventIds of the runtime's marker types */ ThreadIndices thread_idx; + + /** + * TraCR channel of the main thread (workers use 1 + their worker index) + */ + static constexpr uint16_t MAIN_CHANNEL_ID = 0; + + /** + * Whether TraCR is tracing this runtime (requires the pthreads worker backend) + */ + bool _instrumentationActive = false; + + /** + * TraCR channel of the worker running on this thread (set by the worker loops) + */ + inline static thread_local uint16_t _tl_channelId = MAIN_CHANNEL_ID; + + /** + * The last worker state recorded on this thread (used to skip repeated records) + */ + inline static thread_local uint16_t _tl_lastEventId = UINT16_MAX; }; // class Runtime } // namespace taskr diff --git a/meson.build b/meson.build index 7f72229..dfa3aae 100644 --- a/meson.build +++ b/meson.build @@ -80,7 +80,11 @@ if get_option('buildInstrumentation') InstrumentationBuildDep = InstrumentationProject.get_variable('InstrumentationBuildDep') taskrDependencies += InstrumentationBuildDep - add_project_arguments('-DENABLE_INSTRUMENTATION', language: 'cpp') + add_project_arguments('-DENABLE_TRACR', language: 'cpp') + + # Wrap around (overwriting the oldest entries) when a trace buffer is full, + # instead of aborting the application (TraCR's default policy) + add_project_arguments('-DTRACR_POLICY_PERIODIC', language: 'cpp') # uncomment this to enable debug prints of TraCR # add_project_arguments('-DENABLE_DEBUG', language: 'cpp')