From c7fef4266df0aa4a332a904325b61f9906a44835 Mon Sep 17 00:00:00 2001 From: JackTan25 Date: Wed, 1 Jul 2026 17:21:02 +0800 Subject: [PATCH] refactor: dump model inputs as pt files --- rtp_llm/cpp/config/ConfigModules.cc | 3 +- rtp_llm/cpp/config/ConfigModules.h | 1 + rtp_llm/cpp/models/BUILD | 23 ++ rtp_llm/cpp/models/ModelInputsLogger.cc | 272 ++++++++++++++++++ rtp_llm/cpp/models/ModelInputsLogger.h | 24 ++ rtp_llm/cpp/models/PyWrappedModel.cc | 14 +- rtp_llm/cpp/models/PyWrappedModel.h | 46 +-- rtp_llm/cpp/models/test/BUILD | 32 ++- .../models/test/ModelInputsLoggerDumpTool.cc | 54 ++++ .../test/model_inputs_logger_replay_test.py | 231 +++++++++++++++ .../NormalBatchStreamProcessor.cc | 1 + rtp_llm/cpp/normal_engine/NormalExecutor.cc | 9 +- rtp_llm/cpp/normal_engine/NormalExecutor.h | 2 + .../normal_engine/NormalModelInputGatherer.cc | 14 + .../normal_engine/NormalModelInputGatherer.h | 1 + .../normal_engine/speculative/MtpExecutor.cc | 31 +- .../normal_engine/speculative/MtpExecutor.h | 5 +- rtp_llm/cpp/pybind/ConfigInit.cc | 14 +- rtp_llm/models_py/bindings/core/OpData.h | 4 + .../profile_debug_logging_group_args.py | 8 + 20 files changed, 752 insertions(+), 37 deletions(-) create mode 100644 rtp_llm/cpp/models/ModelInputsLogger.cc create mode 100644 rtp_llm/cpp/models/ModelInputsLogger.h create mode 100644 rtp_llm/cpp/models/test/ModelInputsLoggerDumpTool.cc create mode 100644 rtp_llm/cpp/models/test/model_inputs_logger_replay_test.py diff --git a/rtp_llm/cpp/config/ConfigModules.cc b/rtp_llm/cpp/config/ConfigModules.cc index 0d95899850..2daa0aac0e 100644 --- a/rtp_llm/cpp/config/ConfigModules.cc +++ b/rtp_llm/cpp/config/ConfigModules.cc @@ -153,7 +153,8 @@ std::string ProfilingDebugLoggingConfig::to_string() const { << "hack_layer_num: " << hack_layer_num << "\n" << "debug_start_fake_process: " << debug_start_fake_process << "\n" << "enable_detail_log: " << enable_detail_log << "\n" - << "check_nan: " << check_nan << "\n"; + << "check_nan: " << check_nan << "\n" + << "enable_model_inputs_log: " << enable_model_inputs_log << "\n"; return oss.str(); } diff --git a/rtp_llm/cpp/config/ConfigModules.h b/rtp_llm/cpp/config/ConfigModules.h index 25f7623847..bf3364bbdb 100644 --- a/rtp_llm/cpp/config/ConfigModules.h +++ b/rtp_llm/cpp/config/ConfigModules.h @@ -210,6 +210,7 @@ struct ProfilingDebugLoggingConfig { bool debug_start_fake_process = false; bool enable_detail_log = false; bool check_nan = false; + bool enable_model_inputs_log = false; std::string to_string() const; }; diff --git a/rtp_llm/cpp/models/BUILD b/rtp_llm/cpp/models/BUILD index c3afc82afd..d4ac230ec8 100644 --- a/rtp_llm/cpp/models/BUILD +++ b/rtp_llm/cpp/models/BUILD @@ -1,4 +1,5 @@ load("//:def.bzl", "copts") +load("@arch_config//:arch_select.bzl", "torch_deps") # Sample infos - model sampling data structures cc_library( @@ -109,6 +110,26 @@ cc_library( visibility = ["//visibility:public"], ) +cc_library( + name = "model_inputs_logger", + srcs = [ + "ModelInputsLogger.cc", + ], + hdrs = [ + "ModelInputsLogger.h", + ], + deps = [ + "//rtp_llm/cpp/metrics", + "//rtp_llm/cpp/utils:core_utils", + "//rtp_llm/cpp/utils:profiling_scope", + "//rtp_llm/models_py/bindings/core:op_data", + "@havenask//aios/autil:env_util", + "@havenask//aios/autil:time", + ] + torch_deps(), + copts = copts(), + visibility = ["//visibility:public"], +) + cc_library( name = "models", hdrs = glob([ @@ -127,9 +148,11 @@ cc_library( "//rtp_llm/cpp/engine_base/stream:complete_token_ids", "//rtp_llm/cpp/cache:cache_types", "//rtp_llm/cpp/cache:cache", + "//rtp_llm/cpp/metrics", "//rtp_llm/models_py/bindings/core:cache_store_async_writer", "//rtp_llm/cpp/utils:debug_utils", "//rtp_llm/cpp/utils:profiling_scope", + "//rtp_llm/cpp/utils:tensor_debug_utils", "//rtp_llm/cpp/cuda_graph:cuda_graph_impl", "//rtp_llm/cpp/cuda_graph:cuda_graph_base", "//rtp_llm/cpp/cuda_graph:cuda_graph_hdrs_lib", diff --git a/rtp_llm/cpp/models/ModelInputsLogger.cc b/rtp_llm/cpp/models/ModelInputsLogger.cc new file mode 100644 index 0000000000..719986c734 --- /dev/null +++ b/rtp_llm/cpp/models/ModelInputsLogger.cc @@ -0,0 +1,272 @@ +#include "rtp_llm/cpp/models/ModelInputsLogger.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "autil/EnvUtil.h" +#include "autil/TimeUtility.h" +#include "rtp_llm/cpp/utils/Logger.h" +#include "rtp_llm/cpp/utils/ProfilingScope.h" +#include + +namespace rtp_llm { + +namespace { + +constexpr size_t kModelInputsDumpMaxBytes = 100ULL * 1024ULL * 1024ULL; +constexpr size_t kModelInputsDumpFlushThreshold = 100; +constexpr int64_t kModelInputsDumpFlushIntervalUs = 100 * 1000; +constexpr size_t kRecordLengthBytes = sizeof(uint64_t); + +std::string timestampForFile(int64_t now_us) { + std::ostringstream us; + us << std::setfill('0') << std::setw(6) << (now_us % 1000000); + return autil::TimeUtility::usFormat(now_us, "%Y%m%d_%H%M%S") + "_" + us.str(); +} + +torch::Tensor tensorForDump(const torch::Tensor& tensor) { + if (!tensor.defined()) { + return {}; + } + auto out = tensor.detach(); + if (out.scalar_type() == torch::kFloat8_e4m3fn) { + out = out.view(torch::kChar); + } + return out.contiguous(); +} + +void addTensor(c10::impl::GenericDict& payload, + const std::string& name, + const torch::Tensor& tensor, + const torch::Tensor& host_snapshot = {}) { + RTP_LLM_PROFILE_SCOPE_DYNAMIC("model_inputs.dump.tensor(%s)", name.c_str()); + const auto& source = host_snapshot.defined() ? host_snapshot : tensor; + if (!source.defined()) { + payload.insert(name, c10::IValue()); + return; + } + if (source.is_cuda()) { + payload.insert(name, c10::IValue()); + return; + } + payload.insert(name, tensorForDump(source)); +} + +void addSize(c10::impl::GenericDict& payload, const std::string& name, size_t value) { + payload.insert(name, static_cast(value)); +} + +c10::impl::GenericDict buildModelInputsPayload(const GptModelInputs& inputs) { + c10::impl::GenericDict payload(c10::StringType::get(), c10::AnyType::get()); + payload.reserve(96); + + payload.insert("trace_ids", inputs.trace_ids); + + addTensor(payload, "combo_tokens", inputs.combo_tokens, inputs.combo_tokens_host_for_log); + addTensor(payload, "input_lengths", inputs.input_lengths, inputs.input_lengths_host_for_log); + addTensor(payload, "sequence_lengths", inputs.sequence_lengths, inputs.sequence_lengths_host_for_log); + addTensor(payload, "lm_output_indexes", inputs.lm_output_indexes); + addTensor(payload, "prefix_lengths", inputs.prefix_lengths, inputs.prefix_lengths_host_for_log); + addTensor(payload, "combo_tokens_type_ids", inputs.combo_tokens_type_ids); + addTensor(payload, "combo_position_ids", inputs.combo_position_ids); + addTensor(payload, "last_hidden_states", inputs.last_hidden_states); + addTensor(payload, "attention_mask", inputs.attention_mask); + addTensor(payload, "kv_cache_block_id", inputs.kv_cache_block_id); + addTensor(payload, "kv_cache_layer_to_group", inputs.kv_cache_layer_to_group); + addTensor(payload, "kv_cache_group_types", inputs.kv_cache_group_types); + addTensor(payload, "kv_cache_update_mapping", inputs.kv_cache_update_mapping); + addTensor(payload, "request_id", inputs.request_id); + addTensor(payload, "request_pd_separation", inputs.request_pd_separation); + + addSize(payload, "kv_block_stride_bytes", inputs.kv_block_stride_bytes); + addSize(payload, "kv_scale_stride_bytes", inputs.kv_scale_stride_bytes); + addSize(payload, "seq_size_per_block", inputs.seq_size_per_block); + addSize(payload, "kernel_seq_size_per_block", inputs.kernel_seq_size_per_block); + payload.insert("pd_separation", inputs.pd_separation); + payload.insert("decode_entrance", inputs.decode_entrance); + payload.insert("need_all_logits", inputs.need_all_logits); + payload.insert("need_moe_gating", inputs.need_moe_gating); + payload.insert("warmup", inputs.warmup); + payload.insert("skip_run", inputs.skip_run); + payload.insert("is_fake_stream", inputs.is_fake_stream); + payload.insert("is_target_verify", inputs.is_target_verify); + + return payload; +} + +} // namespace + +class ModelInputsLogger::Writer { +public: + Writer(int64_t rank_id, int backup_count): backup_count_(std::max(backup_count, 0)) { + const auto log_path = autil::EnvUtil::getEnv("LOG_PATH", std::string("logs")); + const auto server_id = autil::EnvUtil::getEnv("FRONTEND_SERVER_ID", 0); + prefix_ = "model_inputs_r" + std::to_string(rank_id) + "_s" + std::to_string(server_id); + output_dir_ = std::filesystem::path(log_path); + + std::error_code ec; + std::filesystem::create_directories(output_dir_, ec); + if (ec) { + RTP_LLM_LOG_WARNING("Failed to create model inputs dump directory %s: %s", + output_dir_.string().c_str(), + ec.message().c_str()); + return; + } + + file_path_ = output_dir_ / (prefix_ + ".pt"); + bytes_ = std::filesystem::exists(file_path_, ec) ? std::filesystem::file_size(file_path_, ec) : 0; + open(std::ios::out | std::ios::binary | std::ios::app); + } + + void write(const std::vector& record) { + if (!valid_) { + return; + } + std::lock_guard guard(mutex_); + rotateIfNeeded(kRecordLengthBytes + record.size()); + + const uint64_t record_size = record.size(); + output_.write(reinterpret_cast(&record_size), sizeof(record_size)); + output_.write(record.data(), record.size()); + bytes_ += kRecordLengthBytes + record.size(); + + pending_records_++; + const auto now_us = autil::TimeUtility::currentTimeInMicroSeconds(); + if (pending_records_ >= kModelInputsDumpFlushThreshold + || now_us - last_flush_us_ >= kModelInputsDumpFlushIntervalUs) { + output_.flush(); + pending_records_ = 0; + last_flush_us_ = now_us; + } + } + +private: + void open(std::ios_base::openmode mode) { + output_.open(file_path_, mode); + if (!output_.is_open()) { + RTP_LLM_LOG_WARNING("Failed to open model inputs dump file %s", file_path_.c_str()); + valid_ = false; + return; + } + valid_ = true; + } + + void rotateIfNeeded(size_t next_bytes) { + if (bytes_ == 0 || bytes_ + next_bytes <= kModelInputsDumpMaxBytes) { + return; + } + + output_.close(); + const auto rotated_path = + output_dir_ / (prefix_ + "_" + timestampForFile(autil::TimeUtility::currentTimeInMicroSeconds()) + ".pt"); + std::error_code ec; + std::filesystem::rename(file_path_, rotated_path, ec); + if (ec) { + RTP_LLM_LOG_WARNING("Failed to rotate model inputs dump file from %s to %s: %s", + file_path_.c_str(), + rotated_path.c_str(), + ec.message().c_str()); + } + cleanupOldFiles(); + bytes_ = 0; + pending_records_ = 0; + open(std::ios::out | std::ios::binary | std::ios::trunc); + } + + void cleanupOldFiles() { + if (backup_count_ <= 0) { + return; + } + + std::vector rotated_files; + std::error_code ec; + for (const auto& entry : std::filesystem::directory_iterator(output_dir_, ec)) { + if (ec || !entry.is_regular_file()) { + continue; + } + const auto name = entry.path().filename().string(); + if (name.rfind(prefix_ + "_", 0) == 0 && entry.path().extension() == ".pt") { + rotated_files.push_back(entry); + } + } + if (rotated_files.size() <= static_cast(backup_count_)) { + return; + } + std::sort(rotated_files.begin(), rotated_files.end(), [](const auto& lhs, const auto& rhs) { + return lhs.path().filename().string() < rhs.path().filename().string(); + }); + const auto remove_count = rotated_files.size() - static_cast(backup_count_); + for (size_t i = 0; i < remove_count; ++i) { + std::filesystem::remove(rotated_files[i], ec); + } + } + + std::mutex mutex_; + std::filesystem::path output_dir_; + std::filesystem::path file_path_; + std::ofstream output_; + std::string prefix_; + int backup_count_ = 0; + size_t bytes_ = 0; + size_t pending_records_ = 0; + int64_t last_flush_us_ = 0; + bool valid_ = false; +}; + +ModelInputsLogger::ModelInputsLogger(int64_t rank_id, int backup_count, kmonitor::MetricsReporterPtr metrics_reporter): + metrics_reporter_(std::move(metrics_reporter)), writer_(std::make_unique(rank_id, backup_count)) {} + +ModelInputsLogger::~ModelInputsLogger() = default; + +void ModelInputsLogger::log(const GptModelInputs& inputs) { + if (inputs.is_fake_stream) { + return; + } + + RTP_LLM_PROFILE_SCOPE("model_inputs.dump.total"); + const auto total_start_us = autil::TimeUtility::currentTimeInMicroSeconds(); + try { + c10::impl::GenericDict payload(c10::StringType::get(), c10::AnyType::get()); + { + RTP_LLM_PROFILE_SCOPE("model_inputs.dump.build_payload"); + payload = buildModelInputsPayload(inputs); + } + std::vector pickled; + { + RTP_LLM_PROFILE_SCOPE("model_inputs.dump.pickle_save"); + pickled = torch::pickle_save(c10::IValue(std::move(payload))); + } + { + RTP_LLM_PROFILE_SCOPE("model_inputs.dump.write_file"); + writer_->write(pickled); + } + } catch (const std::exception& e) { + RTP_LLM_LOG_WARNING("Failed to dump model inputs: %s", e.what()); + } catch (...) { + RTP_LLM_LOG_WARNING("Failed to dump model inputs: unknown exception"); + } + + const auto total_us = autil::TimeUtility::currentTimeInMicroSeconds() - total_start_us; + try { + if (metrics_reporter_) { + RTP_LLM_PROFILE_SCOPE("model_inputs.dump.report_metric"); + metrics_reporter_->report( + total_us, "rtp_llm_model_inputs_log_us", kmonitor::MetricType::GAUGE, nullptr, true); + } + } catch (const std::exception& e) { + RTP_LLM_LOG_WARNING("Failed to report model inputs dump metric: %s", e.what()); + } catch (...) { + RTP_LLM_LOG_WARNING("Failed to report model inputs dump metric: unknown exception"); + } +} + +} // namespace rtp_llm diff --git a/rtp_llm/cpp/models/ModelInputsLogger.h b/rtp_llm/cpp/models/ModelInputsLogger.h new file mode 100644 index 0000000000..837c903ca4 --- /dev/null +++ b/rtp_llm/cpp/models/ModelInputsLogger.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include "kmonitor/client/MetricsReporter.h" +#include "rtp_llm/models_py/bindings/core/OpData.h" + +namespace rtp_llm { + +class ModelInputsLogger { +public: + ModelInputsLogger(int64_t rank_id, int backup_count, kmonitor::MetricsReporterPtr metrics_reporter); + ~ModelInputsLogger(); + + void log(const GptModelInputs& inputs); + +private: + class Writer; + + kmonitor::MetricsReporterPtr metrics_reporter_; + std::unique_ptr writer_; +}; + +} // namespace rtp_llm diff --git a/rtp_llm/cpp/models/PyWrappedModel.cc b/rtp_llm/cpp/models/PyWrappedModel.cc index a6bb94f83b..12d51d7330 100644 --- a/rtp_llm/cpp/models/PyWrappedModel.cc +++ b/rtp_llm/cpp/models/PyWrappedModel.cc @@ -4,9 +4,9 @@ #include "rtp_llm/cpp/utils/DebugUtils.h" #include "rtp_llm/cpp/utils/utils.h" #include "rtp_llm/cpp/model_utils/AttentionConfig.h" +#include #include #include -#include #include #include "rtp_llm/cpp/pybind/PyUtils.h" #include "rtp_llm/cpp/utils/AssertUtils.h" @@ -14,6 +14,8 @@ #include #include #include +#include +#include "rtp_llm/cpp/models/ModelInputsLogger.h" #include "rtp_llm/cpp/utils/DevicePerfWrapper.h" #include "rtp_llm/cpp/utils/ProfilingScope.h" #if USING_CUDA @@ -138,7 +140,7 @@ torch_ext::PyAttentionInputs PyWrappedModel::buildPyAttentionInputs(const GptMod py_attn_inputs.context_total_kv_length = cu_kv_seqlens[context_batch_size].item(); py_attn_inputs.total_tokens = cu_seqlens[batch_size].item(); - py_attn_inputs.cu_seqlens = cu_seqlens; + py_attn_inputs.cu_seqlens = cu_seqlens; py_attn_inputs.cu_seqlens_device = tensorHoldHostAndToCuda(cu_seqlens); py_attn_inputs.cu_kv_seqlens_device = tensorHoldHostAndToCuda(cu_kv_seqlens); } else { @@ -154,7 +156,7 @@ torch_ext::PyAttentionInputs PyWrappedModel::buildPyAttentionInputs(const GptMod py_attn_inputs.sequence_lengths.size(0) + 1, 1, torch::TensorOptions(torch::kInt32).device(torch::kCPU).pinned_memory(true)); - py_attn_inputs.decode_cu_seqlens = decode_cu_seqlens; + py_attn_inputs.decode_cu_seqlens = decode_cu_seqlens; py_attn_inputs.decode_cu_seqlens_device = tensorHoldHostAndToCuda(decode_cu_seqlens); } @@ -202,7 +204,7 @@ void PyWrappedModel::setupKVCacheForAttentionInputs(torch_ext::PyAttentionInputs // Legacy 2-D fields default to group 0. // NOTE: keep host/device 2-D fields consistent to avoid shape mismatch in CUDA graph replay path. py_attn_inputs.kv_cache_kernel_block_id_device = py_attn_inputs.kv_cache_kernel_block_id_device_by_group[0]; - py_attn_inputs.kv_cache_kernel_block_id = py_attn_inputs.kv_cache_kernel_block_id_by_group[0]; + py_attn_inputs.kv_cache_kernel_block_id = py_attn_inputs.kv_cache_kernel_block_id_by_group[0]; } // Helper function to build BertEmbeddingInputs from GptModelInputs @@ -457,6 +459,10 @@ GptModelOutputs PyWrappedModel::forward(const GptModelInputs& inputs) { d2d_copies_.clear(); DevicePerfWrapper wrapper(enable_device_perf_, "py model forward"); holdInputsHostBuffers(inputs); + if (model_inputs_logger_) { + model_inputs_logger_->log(inputs); + } + if (pinned_check_remaining_ > 0) { --pinned_check_remaining_; } diff --git a/rtp_llm/cpp/models/PyWrappedModel.h b/rtp_llm/cpp/models/PyWrappedModel.h index 06c62a0c7e..d66a3d6cd4 100644 --- a/rtp_llm/cpp/models/PyWrappedModel.h +++ b/rtp_llm/cpp/models/PyWrappedModel.h @@ -5,7 +5,8 @@ #include "rtp_llm/models_py/bindings/core/torch_utils/TypeConvert.h" #include #include -#include +#include +#include #include "rtp_llm/models_py/bindings/core/Types.h" #include "rtp_llm/models_py/bindings/core/DeviceData.h" #include @@ -27,15 +28,17 @@ namespace py = pybind11; namespace rtp_llm { class KVCacheManager; // Forward declaration +class ModelInputsLogger; class PyWrappedModel: public ModelBase { public: // py_instance is `py_model` indeedly. - PyWrappedModel(const GptModelInitParams& params, - py::object py_instance, - bool is_prefill_cuda_graph_mode = false, - bool use_spec_decoding = false, - const std::vector& kv_cache_layer_to_group = {}); + PyWrappedModel(const GptModelInitParams& params, + py::object py_instance, + bool is_prefill_cuda_graph_mode = false, + bool use_spec_decoding = false, + const std::vector& kv_cache_layer_to_group = {}, + std::shared_ptr model_inputs_logger = nullptr); ~PyWrappedModel(); GptModelOutputs forward(const GptModelInputs& inputs) override; @@ -85,14 +88,15 @@ class PyWrappedModel: public ModelBase { torch::Tensor residual_scale_; ModelBufferHolder buffer_holder_; - GraphBase* graph_runner_{nullptr}; - py::object py_model_; - py::object held_attn_pyobj_; - bool enable_cuda_graph_{false}; - bool is_prefill_cuda_graph_mode_{false}; - bool use_spec_decoding_{false}; - bool enable_device_perf_{false}; - bool check_nan_{false}; + GraphBase* graph_runner_{nullptr}; + py::object py_model_; + py::object held_attn_pyobj_; + bool enable_cuda_graph_{false}; + bool is_prefill_cuda_graph_mode_{false}; + bool use_spec_decoding_{false}; + bool enable_device_perf_{false}; + bool check_nan_{false}; + std::shared_ptr model_inputs_logger_; std::unique_ptr context_parallel_processor_{nullptr}; std::unique_ptr cache_store_async_writer_; @@ -106,11 +110,12 @@ class PyWrappedModel: public ModelBase { }; // NOTE(wangyin): constructor can not be compiled correctly when placed in cc file. -inline PyWrappedModel::PyWrappedModel(const GptModelInitParams& params, - py::object py_instance, - bool is_prefill_cuda_graph_mode, - bool use_spec_decoding, - const std::vector& kv_cache_layer_to_group): +inline PyWrappedModel::PyWrappedModel(const GptModelInitParams& params, + py::object py_instance, + bool is_prefill_cuda_graph_mode, + bool use_spec_decoding, + const std::vector& kv_cache_layer_to_group, + std::shared_ptr model_inputs_logger): device_props_(buildExecProperties(params.parallelism_config, params.device_resource_config)), mla_ops_type_(params.mla_ops_type), layer_num_(params.weights.layers.size()), @@ -120,7 +125,8 @@ inline PyWrappedModel::PyWrappedModel(const GptModelInitParams& params, is_prefill_cuda_graph_mode_(is_prefill_cuda_graph_mode), use_spec_decoding_(use_spec_decoding), enable_device_perf_(params.profile_debug_logging_config.enable_device_perf), - check_nan_(params.profile_debug_logging_config.check_nan) { + check_nan_(params.profile_debug_logging_config.check_nan), + model_inputs_logger_(std::move(model_inputs_logger)) { c10::InferenceMode inference_guard(true); diff --git a/rtp_llm/cpp/models/test/BUILD b/rtp_llm/cpp/models/test/BUILD index ca2706d204..315281d144 100644 --- a/rtp_llm/cpp/models/test/BUILD +++ b/rtp_llm/cpp/models/test/BUILD @@ -1,8 +1,12 @@ load("//:def.bzl", "cc_test_wrapper", "copts") -load("@arch_config//:arch_select.bzl", "torch_deps") +load("@arch_config//:arch_select.bzl", "requirement", "torch_deps") cc_test = cc_test_wrapper +requirement([ + "torch", +]) + test_copts = [ "-fno-access-control", ] + copts() @@ -19,6 +23,10 @@ test_deps = [ "@local_config_cuda//cuda:cudart", ] + torch_deps() +model_inputs_logger_deps = [ + "//rtp_llm/cpp/models:model_inputs_logger", +] + torch_deps() + cc_test( name = "model_data_test", srcs = [ @@ -32,3 +40,25 @@ cc_test( }, exec_properties = {'gpu':'H20'}, ) + +cc_binary( + name = "model_inputs_logger_dump_tool", + srcs = [ + "ModelInputsLoggerDumpTool.cc", + ], + copts = test_copts, + deps = model_inputs_logger_deps, +) + +py_test( + name = "model_inputs_logger_replay_test", + srcs = [ + "model_inputs_logger_replay_test.py", + ], + data = [ + ":model_inputs_logger_dump_tool", + ], + deps = [ + ":torch", + ], +) diff --git a/rtp_llm/cpp/models/test/ModelInputsLoggerDumpTool.cc b/rtp_llm/cpp/models/test/ModelInputsLoggerDumpTool.cc new file mode 100644 index 0000000000..bbb828297b --- /dev/null +++ b/rtp_llm/cpp/models/test/ModelInputsLoggerDumpTool.cc @@ -0,0 +1,54 @@ +#include "rtp_llm/cpp/models/ModelInputsLogger.h" + +#include +#include +#include +#include + +namespace rtp_llm { +namespace { + +GptModelInputs makeInputs() { + GptModelInputs inputs; + inputs.trace_ids = {"trace-a", "trace-b"}; + inputs.combo_tokens = torch::tensor({11, 12, 13}, torch::kInt32); + inputs.input_lengths = torch::tensor({3}, torch::kInt32); + inputs.sequence_lengths = torch::tensor({2}, torch::kInt32); + inputs.lm_output_indexes = torch::tensor({2}, torch::kInt32); + inputs.prefix_lengths = torch::tensor({1}, torch::kInt32); + inputs.combo_tokens_type_ids = torch::tensor({0, 0, 1}, torch::kInt32); + inputs.combo_position_ids = torch::tensor({0, 1, 2}, torch::kInt32); + inputs.kv_cache_block_id = torch::tensor({{{7, 8}}}, torch::kInt32); + inputs.kv_cache_layer_to_group = torch::tensor({0}, torch::kInt32); + inputs.kv_cache_group_types = torch::tensor({1}, torch::kInt32); + inputs.kv_cache_update_mapping = torch::tensor({{1, 2}}, torch::kInt32); + inputs.request_id = torch::tensor({12345}, torch::kInt64); + inputs.request_pd_separation = torch::tensor({false}, torch::kBool); + inputs.kv_block_stride_bytes = 4096; + inputs.kv_scale_stride_bytes = 128; + inputs.seq_size_per_block = 64; + inputs.kernel_seq_size_per_block = 32; + inputs.decode_entrance = true; + inputs.need_all_logits = true; + inputs.need_moe_gating = true; + return inputs; +} + +} // namespace +} // namespace rtp_llm + +int main(int argc, char** argv) { + if (argc != 2 && argc != 3) { + std::cerr << "usage: " << argv[0] << " [dump_count]" << std::endl; + return 2; + } + + const auto dump_count = argc == 3 ? std::stoi(argv[2]) : 1; + setenv("LOG_PATH", argv[1], 1); + setenv("FRONTEND_SERVER_ID", "3", 1); + rtp_llm::ModelInputsLogger logger(/*rank_id=*/2, /*backup_count=*/1, nullptr); + for (int i = 0; i < dump_count; ++i) { + logger.log(rtp_llm::makeInputs()); + } + return 0; +} diff --git a/rtp_llm/cpp/models/test/model_inputs_logger_replay_test.py b/rtp_llm/cpp/models/test/model_inputs_logger_replay_test.py new file mode 100644 index 0000000000..7f506b5681 --- /dev/null +++ b/rtp_llm/cpp/models/test/model_inputs_logger_replay_test.py @@ -0,0 +1,231 @@ +import io +import os +import shlex +import struct +import subprocess +import tempfile +import unittest +from pathlib import Path + +import torch + + +def _iter_model_inputs_dumps(path: Path) -> list[Path]: + if path.is_file(): + return [path] + return sorted( + [*path.rglob("*.pt"), *path.rglob("*.ptlog")], + key=lambda p: str(p.relative_to(path)), + ) + + +def _load_model_inputs_dump_records(path: Path) -> list[dict]: + def load_framed_records() -> list[dict]: + records = [] + with path.open("rb") as f: + while True: + length_bytes = f.read(8) + if not length_bytes: + break + if len(length_bytes) != 8: + raise ValueError(f"incomplete record length in {path}") + (record_size,) = struct.unpack(" str: + test_srcdir = Path(os.environ["TEST_SRCDIR"]) + workspace = os.environ.get("TEST_WORKSPACE", "rtp_llm") + candidates = [ + test_srcdir / workspace / relative_path, + test_srcdir / relative_path, + ] + for candidate in candidates: + if candidate.exists(): + return str(candidate) + raise FileNotFoundError(f"runfile not found: {relative_path}") + + +def _is_tensor(value) -> bool: + return isinstance(value, torch.Tensor) + + +def _is_replayable_payload(payload: dict) -> bool: + return ( + not bool(payload.get("warmup", False)) + and not bool(payload.get("skip_run", False)) + and _is_tensor(payload.get("kv_cache_block_id")) + and _is_tensor(payload.get("combo_tokens")) + and _is_tensor(payload.get("input_lengths")) + and _is_tensor(payload.get("sequence_lengths")) + ) + + +def _to_cpu_i32(tensor: torch.Tensor) -> torch.Tensor: + return tensor.to(device="cpu", dtype=torch.int32).contiguous() + + +def _expand_kernel_block_ids( + block_ids: torch.Tensor, kernel_blocks_per_kv_block: int +) -> torch.Tensor: + if kernel_blocks_per_kv_block <= 1: + return block_ids.clone() + offsets = torch.arange(kernel_blocks_per_kv_block, dtype=torch.int32).view( + *([1] * block_ids.dim()), kernel_blocks_per_kv_block + ) + expanded = block_ids.unsqueeze(-1) * kernel_blocks_per_kv_block + offsets + expanded = torch.where( + block_ids.unsqueeze(-1) == -1, torch.full_like(expanded, -1), expanded + ) + return expanded.reshape( + *block_ids.shape[:-1], block_ids.shape[-1] * kernel_blocks_per_kv_block + ) + + +def _pad_last_dim(tensor: torch.Tensor, size: int) -> torch.Tensor: + if tensor.size(-1) == size: + return tensor + padded = torch.zeros(*tensor.shape[:-1], size, dtype=tensor.dtype) + padded[..., : tensor.size(-1)] = tensor + return padded + + +def _derive_kv_cache_kernel_block_id(payload: dict) -> torch.Tensor: + block_id = payload["kv_cache_block_id"] + if not _is_tensor(block_id) or block_id.numel() == 0: + return torch.empty(0, dtype=torch.int32) + + block_id = _to_cpu_i32(block_id) + seq_size_per_block = int(payload["seq_size_per_block"]) + kernel_seq_size_per_block = int(payload["kernel_seq_size_per_block"]) + full_group_bpk = 1 + if kernel_seq_size_per_block > 0: + full_group_bpk = max(1, seq_size_per_block // kernel_seq_size_per_block) + + group_types = payload.get("kv_cache_group_types") + group_types = ( + _to_cpu_i32(group_types).flatten() + if _is_tensor(group_types) + else torch.empty(0, dtype=torch.int32) + ) + + if block_id.dim() != 3: + group_type = int(group_types[0].item()) if group_types.numel() else 1 + return _expand_kernel_block_ids( + block_id, full_group_bpk if group_type == 1 else 1 + ) + + groups = [] + kernel_block_count = block_id.size(-1) * full_group_bpk + for group_idx in range(block_id.size(0)): + group_type = ( + int(group_types[group_idx].item()) if group_idx < group_types.numel() else 1 + ) + group_kernel_blocks = _expand_kernel_block_ids( + block_id[group_idx], full_group_bpk if group_type == 1 else 1 + ) + groups.append(_pad_last_dim(group_kernel_blocks, kernel_block_count)) + return torch.stack(groups, dim=0) + + +def _validate_model_inputs_dump(path: Path) -> dict: + records = _load_model_inputs_dump_records(path) + assert len(records) > 0 + payload = records[0] + expected_keys = { + "trace_ids", + "combo_tokens", + "input_lengths", + "sequence_lengths", + "lm_output_indexes", + "prefix_lengths", + "combo_tokens_type_ids", + "combo_position_ids", + "last_hidden_states", + "attention_mask", + "kv_cache_block_id", + "kv_cache_layer_to_group", + "kv_cache_group_types", + "kv_cache_update_mapping", + "request_id", + "request_pd_separation", + "kv_block_stride_bytes", + "kv_scale_stride_bytes", + "seq_size_per_block", + "kernel_seq_size_per_block", + "pd_separation", + "decode_entrance", + "need_all_logits", + "need_moe_gating", + "warmup", + "skip_run", + "is_fake_stream", + "is_target_verify", + } + assert set(payload.keys()) == expected_keys + assert payload["combo_tokens"].tolist() == [11, 12, 13] + assert payload["last_hidden_states"] is None + assert payload["attention_mask"] is None + assert tuple(payload["kv_cache_block_id"].shape) == (1, 1, 2) + kernel_block_id = _derive_kv_cache_kernel_block_id(payload) + assert tuple(kernel_block_id.shape) == (1, 1, 4) + assert kernel_block_id.tolist() == [[[14, 15, 16, 17]]] + return payload + + +class ModelInputsLoggerReplayTest(unittest.TestCase): + def test_generated_dump_can_be_loaded_by_torch(self) -> None: + tool = _runfile_path("rtp_llm/cpp/models/test/model_inputs_logger_dump_tool") + with tempfile.TemporaryDirectory() as temp_dir: + subprocess.check_call([tool, temp_dir, "2"]) + dumps = sorted(Path(temp_dir).glob("model_inputs_r2_s3*.pt")) + self.assertEqual(1, len(dumps)) + records = _load_model_inputs_dump_records(dumps[0]) + self.assertEqual(2, len(records)) + payload = _validate_model_inputs_dump(dumps[0]) + self.assertEqual(["trace-a", "trace-b"], payload["trace_ids"]) + + def test_optional_external_dump_forward_replay(self) -> None: + replay_path = os.environ.get("MODEL_INPUTS_DUMP_PATH") + if not replay_path: + self.skipTest("MODEL_INPUTS_DUMP_PATH is not set") + + path = Path(replay_path) + dumps = _iter_model_inputs_dumps(path) + self.assertGreater(len(dumps), 0) + replayable_count = 0 + for dump in dumps: + for payload in _load_model_inputs_dump_records(dump): + if not _is_replayable_payload(payload): + continue + replayable_count += 1 + self.assertTrue(_is_tensor(_derive_kv_cache_kernel_block_id(payload))) + self.assertGreater(replayable_count, 0) + + replay_cmd = os.environ.get("MODEL_INPUTS_FORWARD_REPLAY_CMD") + if replay_cmd: + env = os.environ.copy() + env["MODEL_INPUTS_DUMP_PATH"] = str(path) + subprocess.check_call(shlex.split(replay_cmd), env=env) + + +if __name__ == "__main__": + unittest.main() diff --git a/rtp_llm/cpp/normal_engine/NormalBatchStreamProcessor.cc b/rtp_llm/cpp/normal_engine/NormalBatchStreamProcessor.cc index f628d2910d..f784cd9caa 100644 --- a/rtp_llm/cpp/normal_engine/NormalBatchStreamProcessor.cc +++ b/rtp_llm/cpp/normal_engine/NormalBatchStreamProcessor.cc @@ -28,6 +28,7 @@ NormalBatchStreamProcessor::NormalBatchStreamProcessor( model_input_gatherer_config_.kv_cache_group_types = cache_config.group_types; model_input_gatherer_config_.warm_up = warm_up; model_input_gatherer_config_.enable_detail_log = profiling_debug_logging_config.enable_detail_log; + model_input_gatherer_config_.enable_model_inputs_log = profiling_debug_logging_config.enable_model_inputs_log; model_input_gatherer_ = std::make_unique(model_input_gatherer_config_); sampler_input_gatherer_ = std::make_unique(); diff --git a/rtp_llm/cpp/normal_engine/NormalExecutor.cc b/rtp_llm/cpp/normal_engine/NormalExecutor.cc index 663b8bc2d8..3e0e5ca13d 100644 --- a/rtp_llm/cpp/normal_engine/NormalExecutor.cc +++ b/rtp_llm/cpp/normal_engine/NormalExecutor.cc @@ -5,6 +5,7 @@ #include #include "rtp_llm/cpp/utils/StatusUtil.h" #include "rtp_llm/cpp/utils/ProfilingScope.h" +#include "rtp_llm/cpp/models/ModelInputsLogger.h" #include "rtp_llm/cpp/models/ModelTypes.h" #include "rtp_llm/cpp/models/PyWrappedModel.h" #include "rtp_llm/cpp/models/Sampler.h" @@ -41,6 +42,12 @@ NormalExecutor::NormalExecutor(const EngineInitParams& params, tp_rank_ = params.parallelism_config.tp_rank; parallelism_config_ = params.parallelism_config; RTP_LLM_LOG_INFO("enable_detail_log_ = %d, tp_rank_ = %d", enable_detail_log_, tp_rank_); + if (params.profiling_debug_logging_config.enable_model_inputs_log) { + model_inputs_logger_ = + std::make_shared(params.parallelism_config.world_rank, + params.profiling_debug_logging_config.log_file_backup_count, + metrics_reporter_); + } if (params.eplb_config.enable_eplb() && params.model_config_.moe_style != 0) { // use first moe layer weight as moe weight type @@ -96,7 +103,7 @@ NormalExecutor::NormalExecutor(const EngineInitParams& params, } if (!params.py_model.is_none()) { RTP_LLM_LOG_INFO("init executor with python model"); - model_.reset(new PyWrappedModel(model_init_params, params.py_model)); + model_.reset(new PyWrappedModel(model_init_params, params.py_model, false, false, {}, model_inputs_logger_)); } else if (test_model_factory) { RTP_LLM_LOG_INFO("init executor with test model factory"); model_ = test_model_factory(model_init_params); diff --git a/rtp_llm/cpp/normal_engine/NormalExecutor.h b/rtp_llm/cpp/normal_engine/NormalExecutor.h index 4285b907be..7305fa60df 100644 --- a/rtp_llm/cpp/normal_engine/NormalExecutor.h +++ b/rtp_llm/cpp/normal_engine/NormalExecutor.h @@ -16,6 +16,7 @@ namespace rtp_llm { class KVCacheManager; +class ModelInputsLogger; struct GptModelInitParams; class NormalExecutor: public Executor { @@ -53,6 +54,7 @@ class NormalExecutor: public Executor { std::unique_ptr sampler_; std::unique_ptr batch_stream_processor_; std::shared_ptr cache_manager_; + std::shared_ptr model_inputs_logger_; std::shared_ptr expert_balancer_; bool warm_up_; bool use_all_gather_; diff --git a/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.cc b/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.cc index 2d83fd50cd..ce916647c3 100644 --- a/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.cc +++ b/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.cc @@ -409,6 +409,9 @@ absl::Status NormalModelInputGatherer::processContextStreams(GptModelInputs& && ctx.mm_feature_index < model_input.mm_features_locs.numel()) { model_input.mm_features_locs = model_input.mm_features_locs.slice(0, 0, ctx.mm_feature_index); } + if (config_.enable_model_inputs_log) { + model_input.prefix_lengths_host_for_log = model_input.prefix_lengths; + } return absl::OkStatus(); } @@ -421,6 +424,17 @@ absl::StatusOr NormalModelInputGatherer::gather(const StreamGrou initializeKvCacheMetadata(model_input); RETURN_IF_STATUS_ERROR(processDecodeStreams(model_input, stream_groups)); RETURN_IF_STATUS_ERROR(processContextStreams(model_input, stream_groups)); + if (config_.enable_model_inputs_log) { + if (model_input.combo_tokens.defined() && !model_input.combo_tokens.is_cuda()) { + model_input.combo_tokens_host_for_log = model_input.combo_tokens; + } + if (model_input.input_lengths.defined() && !model_input.input_lengths.is_cuda()) { + model_input.input_lengths_host_for_log = model_input.input_lengths; + } + if (model_input.sequence_lengths.defined() && !model_input.sequence_lengths.is_cuda()) { + model_input.sequence_lengths_host_for_log = model_input.sequence_lengths; + } + } return model_input; } diff --git a/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.h b/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.h index ea638ff49a..647ca7dccc 100644 --- a/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.h +++ b/rtp_llm/cpp/normal_engine/NormalModelInputGatherer.h @@ -33,6 +33,7 @@ struct NormalModelInputGathererConfig { std::vector kv_cache_group_types; bool warm_up{}; bool enable_detail_log{}; + bool enable_model_inputs_log{}; }; class NormalModelInputGatherer { diff --git a/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.cc b/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.cc index 9d9a1fe79d..69960887b2 100644 --- a/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.cc +++ b/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.cc @@ -12,6 +12,7 @@ #include "rtp_llm/cpp/utils/Logger.h" #include "rtp_llm/cpp/utils/AssertUtils.h" #include "rtp_llm/cpp/utils/StringUtil.h" +#include "rtp_llm/cpp/models/ModelInputsLogger.h" #include "rtp_llm/cpp/models/PyWrappedModel.h" #include "rtp_llm/cpp/models/logits_processor/LogitsProcessorFactory.h" #include "rtp_llm/cpp/utils/ProfilingScope.h" @@ -133,6 +134,12 @@ MtpExecutor::MtpExecutor(const EngineInitParams& params, tp_rank_ = params.parallelism_config.tp_rank; parallelism_config_ = params.parallelism_config; RTP_LLM_LOG_INFO("enable_detail_log_ = %d, tp_rank_ = %d", enable_detail_log_, tp_rank_); + if (params.profiling_debug_logging_config.enable_model_inputs_log) { + model_inputs_logger_ = + std::make_shared(params.parallelism_config.world_rank, + params.profiling_debug_logging_config.log_file_backup_count, + metrics_reporter_); + } if (params.eplb_config.enable_eplb() && params.model_config_.moe_style != 0) { // use first moe layer weight as moe weight type @@ -196,8 +203,12 @@ MtpExecutor::MtpExecutor(const EngineInitParams& params, if (!params.py_model.is_none()) { RTP_LLM_LOG_INFO("init executor with python model"); - model_.reset(new PyWrappedModel( - model_init_params, params.py_model, false, true, target_cache_layer_layout.layer_to_groups)); + model_.reset(new PyWrappedModel(model_init_params, + params.py_model, + false, + true, + target_cache_layer_layout.layer_to_groups, + model_inputs_logger_)); } // when warmup, cache manager maybe nullptr @@ -238,8 +249,12 @@ MtpExecutor::MtpExecutor(const EngineInitParams& params, cache_manager}); if (!params.py_sp_model.is_none()) { RTP_LLM_LOG_INFO("[speculative decoding] using py model"); - draft_model_.reset(new PyWrappedModel( - model_params, params.py_sp_model, false, false, draft_cache_layer_layout.layer_to_groups)); + draft_model_.reset(new PyWrappedModel(model_params, + params.py_sp_model, + false, + false, + draft_cache_layer_layout.layer_to_groups, + model_inputs_logger_)); // Create separate model for speculative prefill with CUDA graph if enabled (from params) const bool enable_cuda_graph = params.hw_kernel_config.enable_cuda_graph; RTP_LLM_LOG_INFO( @@ -248,8 +263,12 @@ MtpExecutor::MtpExecutor(const EngineInitParams& params, if (enable_cuda_graph) { RTP_LLM_LOG_INFO( "[speculative decoding] creating separate prefill draft model with CUDA graph support"); - sp_prefill_draft_model_.reset(new PyWrappedModel( - model_params, params.py_sp_model, true, false, draft_cache_layer_layout.layer_to_groups)); + sp_prefill_draft_model_.reset(new PyWrappedModel(model_params, + params.py_sp_model, + true, + false, + draft_cache_layer_layout.layer_to_groups, + model_inputs_logger_)); } } break; // NOTE: only support one mtp model now diff --git a/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.h b/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.h index d3a57e2aee..cdb2225c13 100644 --- a/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.h +++ b/rtp_llm/cpp/normal_engine/speculative/MtpExecutor.h @@ -15,6 +15,8 @@ namespace rtp_llm { +class ModelInputsLogger; + struct MtpMetricsCollector { RtpLLMExecutorMetricsCollector executor_collector; RtpLLMTokenPSMetricsCollector tps_collector; @@ -111,6 +113,7 @@ class MtpExecutor: public Executor { std::unique_ptr sampler_; std::unique_ptr batch_stream_processor_; std::shared_ptr cache_manager_; + std::shared_ptr model_inputs_logger_; bool enable_ffn_disaggregate_ = false; bool enable_detail_log_ = false; int tp_rank_ = 0; @@ -139,4 +142,4 @@ class MtpExecutor: public Executor { torch::Tensor target_kv_cache_layer_to_group; torch::Tensor draft_kv_cache_layer_to_group; }; -}; // namespace rtp_llm \ No newline at end of file +}; // namespace rtp_llm diff --git a/rtp_llm/cpp/pybind/ConfigInit.cc b/rtp_llm/cpp/pybind/ConfigInit.cc index 1c49b9912b..4cf39bc270 100644 --- a/rtp_llm/cpp/pybind/ConfigInit.cc +++ b/rtp_llm/cpp/pybind/ConfigInit.cc @@ -520,6 +520,7 @@ PYBIND11_MODULE(libth_transformer_config, m) { .def_readwrite("debug_start_fake_process", &ProfilingDebugLoggingConfig::debug_start_fake_process) .def_readwrite("enable_detail_log", &ProfilingDebugLoggingConfig::enable_detail_log) .def_readwrite("check_nan", &ProfilingDebugLoggingConfig::check_nan) + .def_readwrite("enable_model_inputs_log", &ProfilingDebugLoggingConfig::enable_model_inputs_log) .def("to_string", &ProfilingDebugLoggingConfig::to_string) .def(py::pickle( [](const ProfilingDebugLoggingConfig& self) { @@ -537,10 +538,11 @@ PYBIND11_MODULE(libth_transformer_config, m) { self.hack_layer_num, self.debug_start_fake_process, self.enable_detail_log, - self.check_nan); + self.check_nan, + self.enable_model_inputs_log); }, [](py::tuple t) { - if (t.size() != 12 && t.size() != 15) + if (t.size() != 12 && t.size() != 13 && t.size() != 15 && t.size() != 16) throw std::runtime_error("Invalid state!"); ProfilingDebugLoggingConfig c; @@ -550,7 +552,7 @@ PYBIND11_MODULE(libth_transformer_config, m) { c.ft_core_dump_on_exception = t[2].cast(); c.ft_alog_conf_path = t[3].cast(); c.gen_timeline_sync = t[4].cast(); - if (t.size() == 12) { + if (t.size() == 12 || t.size() == 13) { c.torch_cuda_profiler_dir = t[5].cast(); c.log_file_backup_count = t[6].cast(); c.debug_load_server = t[7].cast(); @@ -558,6 +560,9 @@ PYBIND11_MODULE(libth_transformer_config, m) { c.debug_start_fake_process = t[9].cast(); c.enable_detail_log = t[10].cast(); c.check_nan = t[11].cast(); + if (t.size() == 13) { + c.enable_model_inputs_log = t[12].cast(); + } } else { c.timeline_start_step = t[5].cast(); c.timeline_num_steps = t[6].cast(); @@ -569,6 +574,9 @@ PYBIND11_MODULE(libth_transformer_config, m) { c.debug_start_fake_process = t[12].cast(); c.enable_detail_log = t[13].cast(); c.check_nan = t[14].cast(); + if (t.size() == 16) { + c.enable_model_inputs_log = t[15].cast(); + } } } catch (const std::exception& e) { throw std::runtime_error(std::string("ProfilingDebugLoggingConfig unpickle error: ") + e.what()); diff --git a/rtp_llm/models_py/bindings/core/OpData.h b/rtp_llm/models_py/bindings/core/OpData.h index ffca5000d6..9e72f76b84 100644 --- a/rtp_llm/models_py/bindings/core/OpData.h +++ b/rtp_llm/models_py/bindings/core/OpData.h @@ -39,6 +39,10 @@ struct GptModelInputs { torch::Tensor lm_output_indexes; // [sum(lm_output_lengths)] torch::Tensor lm_output_lengths; // [total_batch_size] torch::Tensor prefix_lengths; // [context_batch_size] + torch::Tensor combo_tokens_host_for_log; + torch::Tensor input_lengths_host_for_log; + torch::Tensor sequence_lengths_host_for_log; + torch::Tensor prefix_lengths_host_for_log; torch::Tensor combo_tokens_type_ids; // [cumulated_seq_len] torch::Tensor combo_position_ids; // [cumulated_seq_len] diff --git a/rtp_llm/server/server_args/profile_debug_logging_group_args.py b/rtp_llm/server/server_args/profile_debug_logging_group_args.py index 041fe87c72..b2e5e2b92e 100644 --- a/rtp_llm/server/server_args/profile_debug_logging_group_args.py +++ b/rtp_llm/server/server_args/profile_debug_logging_group_args.py @@ -120,3 +120,11 @@ def init_profile_debug_logging_group_args(parser, profiling_debug_config): default=False, help="控制是否check nan, 为了排查。可选值: True (启用), False (禁用)。默认为 False", ) + profile_debug_logging_group.add_argument( + "--enable_model_inputs_log", + env_name="ENABLE_MODEL_INPUTS_LOG", + bind_to=(profiling_debug_config, "enable_model_inputs_log"), + type=str2bool, + default=False, + help="控制是否打印模型输入日志。可选值: True (启用), False (禁用)。默认为 False", + )