Skip to content

Add cooperative cancel for LP/MIP solves #1693

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tmckayus wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add cooperative cancel for LP/MIP solves #1693

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cpp/include/cuopt/mathematical_optimization/constants.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,7 @@
#define CUOPT_TERMINATION_STATUS_CONCURRENT_LIMIT 9
#define CUOPT_TERMINATION_STATUS_WORK_LIMIT 10
#define CUOPT_TERMINATION_STATUS_UNBOUNDED_OR_INFEASIBLE 11
#define CUOPT_TERMINATION_STATUS_CANCELLED 12

/* @brief The objective sense constants */
#define CUOPT_MINIMIZE 1
Expand Down
5 changes: 5 additions & 0 deletions cpp/include/cuopt/mathematical_optimization/mip/solver_settings.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

#pragma once

#include <atomic>
#include <optional>
#include <vector>

Expand Down Expand Up @@ -183,6 +184,10 @@ class mip_solver_settings_t {
// benchmarks
benchmark_info_t* benchmark_info_ptr = nullptr;

// Optional cooperative cancel (level-triggered). Non-owning. Cleared by
// solve_mip on return. Set true from another thread to request early exit.
std::atomic<bool>* cancel_requested{nullptr};

Comment on lines +187 to +190

@coderabbitai coderabbitai Bot Aug 7, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Description: Locate solve_mip implementations and check for cancel_requested reset behavior.
set -euo pipefail
rg -n -C6 'cancel_requested' cpp/src/mip_heuristics/solve.cu cpp/src/branch_and_bound/branch_and_bound.cpp 2>/dev/null || true
rg -n 'cancel_requested->store|\*.*cancel_requested\s*=' cpp/src --type=cpp --type=cuda -g '*.cu' -g '*.cpp' -g '*.hpp'

Repository: NVIDIA/cuopt

Length of output: 11252

🏁 Script executed:

#!/bin/bash
set -euo pipefail

printf '%s\n' '--- solve_mip declarations and definitions ---'
rg -n -C8 'solve_mip|cancel_requested' cpp/include cpp/src --glob '*.{hpp,h,cpp,cu,cuh,inl}' | \
  rg -n 'solve_mip|cancel_requested|catch|return' | head -n 500

printf '%s\n' '--- complete solve.cu control-flow regions ---'
sed -n '200,270p' cpp/src/mip_heuristics/solve.cu
sed -n '630,830p' cpp/src/mip_heuristics/solve.cu
sed -n '830,930p' cpp/src/mip_heuristics/solve.cu

printf '%s\n' '--- all writes or helper implementations involving the cancellation flag ---'
rg -n -C4 'cancel_requested|cancel_flag_set|solve_limit_reached' cpp/include cpp/src --glob '*.{hpp,h,cpp,cu,cuh,inl}' | \
  rg -v 'load\(|get_cancel|cancel_flag_set|solve_limit_reached' | head -n 500

Repository: NVIDIA/cuopt

Length of output: 50368

🏁 Script executed:

#!/bin/bash
set -euo pipefail

printf '%s\n' '--- solve_mip_helper and solve_mip implementations ---'
sed -n '320,355p' cpp/src/mip_heuristics/solve.cu
sed -n '800,885p' cpp/src/mip_heuristics/solve.cu
sed -n '885,980p' cpp/src/mip_heuristics/solve.cu

printf '%s\n' '--- cancellation writes across the repository ---'
rg -n 'cancel_requested[^;\n]*(store|exchange)|cancel_requested[^;\n]*=' cpp/include cpp/src --glob '*.{hpp,h,cpp,cu,cuh,inl}' | head -n 300
rg -n '\.(store|exchange)\([^)]*\)' cpp/src --glob '*.{cpp,cu}' | rg 'cancel|preempt|halt' | head -n 200

printf '%s\n' '--- relevant cancellation helper definitions ---'
rg -n -C8 'cancel_flag_set|atomic_flag_set|remap_limit_status_if_cancelled' cpp/include cpp/src --glob '*.{hpp,h,cpp,cu,cuh,inl}' | head -n 300

Repository: NVIDIA/cuopt

Length of output: 37164

Reset the cancellation flag before returning from solve_mip

solve_mip does not reset *settings.cancel_requested, so reusing a true flag causes the next solve to exit immediately. Reset the pointee on normal, early-return, and exception paths. Document that the pointee value is cleared, not the pointer.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@cpp/include/cuopt/mathematical_optimization/mip/solver_settings.hpp` around
lines 187 - 190, Update the solve_mip cancellation handling so the pointee
behind settings.cancel_requested is cleared before solve_mip returns, including
normal completion, early-return paths, and exception exits. Keep
cancel_requested as a non-owning pointer in solver_settings.hpp, but make the
implementation reset the referenced atomic<bool> value rather than the pointer
itself, and update the related comment/documentation to state that the pointee
value is cleared.

// TODO check with Akif and Alice
pdlp::pdlp_hyper_params_t hyper_params;

Expand Down
2 changes: 2 additions & 0 deletions cpp/include/cuopt/mathematical_optimization/mip/solver_solution.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ enum class mip_termination_status_t : int8_t {
TimeLimit = CUOPT_TERMINATION_STATUS_TIME_LIMIT,
WorkLimit = CUOPT_TERMINATION_STATUS_WORK_LIMIT,
UnboundedOrInfeasible = CUOPT_TERMINATION_STATUS_UNBOUNDED_OR_INFEASIBLE,
Cancelled = CUOPT_TERMINATION_STATUS_CANCELLED,
};

template <typename i_t, typename f_t>
Expand Down Expand Up @@ -65,6 +66,7 @@ class mip_solution_t : public base_solution_t {
double get_total_solve_time() const;
double get_presolve_time() const;
mip_termination_status_t get_termination_status() const;
void set_termination_status(mip_termination_status_t termination_status);
static std::string get_termination_status_string(mip_termination_status_t termination_status);
std::string get_termination_status_string() const;
const cuopt::logic_error& get_error_status() const;
Expand Down
3 changes: 3 additions & 0 deletions cpp/include/cuopt/mathematical_optimization/pdlp/solver_settings.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,9 @@ class pdlp_solver_settings_t {
bool inside_mip{false};
// For concurrent termination
std::atomic<int>* concurrent_halt{nullptr};
// Optional cooperative cancel (level-triggered). Non-owning. Cleared by
// solve_lp on return. Set true from another thread to request early exit.
std::atomic<bool>* cancel_requested{nullptr};
// Shared strong branching solved flags for cooperative DS + PDLP
cuda::std::span<std::atomic<int>> shared_sb_solved;
static constexpr f_t minimal_absolute_tolerance = 1.0e-12;
Expand Down
3 changes: 2 additions & 1 deletion cpp/include/cuopt/mathematical_optimization/pdlp/solver_solution.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ enum class pdlp_termination_status_t : int8_t {
TimeLimit = CUOPT_TERMINATION_STATUS_TIME_LIMIT,
PrimalFeasible = CUOPT_TERMINATION_STATUS_PRIMAL_FEASIBLE,
ConcurrentLimit = CUOPT_TERMINATION_STATUS_CONCURRENT_LIMIT,
UnboundedOrInfeasible = CUOPT_TERMINATION_STATUS_UNBOUNDED_OR_INFEASIBLE
UnboundedOrInfeasible = CUOPT_TERMINATION_STATUS_UNBOUNDED_OR_INFEASIBLE,
Cancelled = CUOPT_TERMINATION_STATUS_CANCELLED
};

/**
Expand Down
113 changes: 40 additions & 73 deletions cpp/src/barrier/barrier.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ class iteration_data_t {
find_dense_columns(
lp.A, settings, dense_columns_unordered, n_dense_rows, max_row_nz, estimated_nz_AAT);
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
#ifdef PRINT_INFO
for (i_t j : dense_columns_unordered) {
settings.log.printf("Dense column %6d\n", j);
Expand Down Expand Up @@ -576,7 +576,7 @@ class iteration_data_t {
if (n_upper_bounds > 0 || (has_Q && !use_augmented)) { inv_diag.sqrt(inv_sqrt_diag); }
}

if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }

{
raft::common::nvtx::range scope("Barrier: LP Data: AD matrix setup");
Expand Down Expand Up @@ -642,7 +642,7 @@ class iteration_data_t {
RAFT_CHECK_CUDA(handle_ptr->get_stream());
}

if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
{
raft::common::nvtx::range scope("Barrier: LP Data: Cholesky init");
i_t factorization_size =
Expand All @@ -651,7 +651,7 @@ class iteration_data_t {
handle_ptr, settings, factorization_size);
chol->set_positive_definite(false);
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
{
raft::common::nvtx::range scope("Barrier: LP Data: symbolic analysis");
// Perform symbolic analysis
Expand All @@ -662,14 +662,14 @@ class iteration_data_t {
// Build the sparsity pattern of the augmented system
form_augmented(true);
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
symbolic_status = chol->analyze(device_augmented);
} else {
{
raft::common::nvtx::range form_scope("Barrier: LP Data: form ADAT");
form_adat(true);
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
symbolic_status = chol->analyze(device_ADAT);
}
}
Expand Down Expand Up @@ -918,7 +918,7 @@ class iteration_data_t {
});
RAFT_CHECK_CUDA(stream_view_);
}
if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return; }
if (settings_.cancel_or_halt_requested()) { return; }
if (first_call) {
raft::common::nvtx::range scope("Barrier: Form ADAT: cusparse init");
try {
Expand All @@ -929,7 +929,7 @@ class iteration_data_t {
return;
}
}
if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) { return; }
if (settings_.cancel_or_halt_requested()) { return; }

{
raft::common::nvtx::range scope("Barrier: Form ADAT: ADAT multiply");
Expand Down Expand Up @@ -1023,9 +1023,7 @@ class iteration_data_t {
dense_vector_t<i_t, f_t> M_col(AD.m);
solve_status = chol->solve(U_col, M_col);
if (solve_status != 0) { return solve_status; }
if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) {
return CONCURRENT_HALT_RETURN;
}
if (settings_.cancel_or_halt_requested()) { return CONCURRENT_HALT_RETURN; }
M.set_column(k, M_col);

if (debug) {
Expand All @@ -1042,9 +1040,7 @@ class iteration_data_t {
for (i_t k = 0; k < n_dense_columns; k++) {
AD_dense.transpose_multiply(
1.0, M.values.data() + k * M.m, 0.0, H.values.data() + k * H.m);
if (settings_.concurrent_halt != nullptr && *settings_.concurrent_halt == 1) {
return CONCURRENT_HALT_RETURN;
}
if (settings_.cancel_or_halt_requested()) { return CONCURRENT_HALT_RETURN; }
}

dense_vector_t<i_t, f_t> e(n_dense_columns);
Expand Down Expand Up @@ -1442,7 +1438,7 @@ class iteration_data_t {
std::sort(column_nz_permutation.begin(),
column_nz_permutation.end(),
[&column_nz](i_t i, i_t j) { return column_nz[i] < column_nz[j]; });
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }

// We then compute the exact sparsity pattern for columns of A whose where
// the number of nonzeros is less than a threshold. This part can be done
Expand Down Expand Up @@ -1473,7 +1469,7 @@ class iteration_data_t {
// The best way to do that is to have A stored in CSR format.
csr_matrix_t<i_t, f_t> A_row(0, 0, 0);
A.to_compressed_row(A_row);
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }

std::vector<i_t> histogram(m + 1, 0);
for (i_t j = 0; j < n; j++) {
Expand Down Expand Up @@ -1545,7 +1541,7 @@ class iteration_data_t {
delta_nz[j] +=
fill; // Capture contributions from A(:, j). j will be encountered multiple times
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
}

int64_t sparse_nz_C = 0;
Expand Down Expand Up @@ -1585,7 +1581,7 @@ class iteration_data_t {
delta_nz[j] + static_cast<int64_t>(
fill_estimate)); // Capture the estimated fill associated with column j
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }
}

int64_t estimated_nz_C = 0;
Expand All @@ -1603,7 +1599,7 @@ class iteration_data_t {
std::sort(permutation.begin(), permutation.end(), [&delta_nz](i_t i, i_t j) {
return delta_nz[i] < delta_nz[j];
});
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) { return; }
if (settings.cancel_or_halt_requested()) { return; }

// Now we make a forward pass and compute the number of nonzeros in C
// assuming we had included column j
Expand Down Expand Up @@ -2786,9 +2782,7 @@ i_t barrier_solver_t<i_t, f_t>::gpu_compute_search_direction(iteration_data_t<i_
// Check halt after form_augmented (synchronous) and before factorize (~1s).
// If halt was set while form_augmented ran, we catch it here and skip the
// expensive factorization entirely.
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
return CONCURRENT_HALT_RETURN;
}
if (settings.cancel_or_halt_requested()) { return CONCURRENT_HALT_RETURN; }
{
raft::common::nvtx::range fun_scope("Barrier: factorize");
status = data.chol->factorize(data.device_augmented);
Expand All @@ -2806,9 +2800,7 @@ i_t barrier_solver_t<i_t, f_t>::gpu_compute_search_direction(iteration_data_t<i_
// Check halt after form_adat (synchronous) and before factorize (~1s).
// If halt was set while form_adat ran, we catch it here and skip the
// expensive Cholesky factorization entirely.
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
return CONCURRENT_HALT_RETURN;
}
if (settings.cancel_or_halt_requested()) { return CONCURRENT_HALT_RETURN; }
{
raft::common::nvtx::range fun_scope("Barrier: factorize");
status = data.chol->factorize(data.device_ADAT);
Expand Down Expand Up @@ -4102,6 +4094,21 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
try {
raft::common::nvtx::range fun_scope("Barrier: solve");

auto barrier_limit_status = [&]() -> std::optional<lp_status_t> {
const auto limit = settings.check_solve_limits(start_time);
if (limit == cuopt::solve_limit_reason_t::None) { return std::nullopt; }
if (limit == cuopt::solve_limit_reason_t::Cancelled) {
settings.log.printf("Barrier solve cancelled\n");
return lp_status_t::CANCELLED;
}
if (limit == cuopt::solve_limit_reason_t::ConcurrentHalt) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
settings.log.printf("Barrier time limit exceeded\n");
return lp_status_t::TIME_LIMIT;
};

i_t n = lp.num_cols;
i_t m = lp.num_rows;

Expand Down Expand Up @@ -4136,10 +4143,7 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,

iteration_data_t<i_t, f_t> data(
lp, num_upper_bounds, presolve_info.direct_free_variables, Q, settings);
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }
if (data.indefinite_Q) { return lp_status_t::NUMERICAL_ISSUES; }
if (data.symbolic_status != 0) {
settings.log.printf("Error in symbolic analysis\n");
Expand All @@ -4155,20 +4159,10 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
data.cusparse_y_residual_ = data.cusparse_view_.create_vector(data.d_y_residual_);
data.restrict_u_.resize(num_upper_bounds);

if (toc(start_time) > settings.time_limit) {
settings.log.printf("Barrier time limit exceeded\n");
return lp_status_t::TIME_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }

i_t initial_status = initial_point(data);
if (toc(start_time) > settings.time_limit) {
settings.log.printf("Barrier time limit exceeded\n");
return lp_status_t::TIME_LIMIT;
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }
if (initial_status != 0) {
settings.log.printf("Unable to compute initial point\n");
return lp_status_t::NUMERICAL_ISSUES;
Expand Down Expand Up @@ -4274,14 +4268,7 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
while (iter < iteration_limit) {
raft::common::nvtx::range fun_scope("Barrier: iteration");

if (toc(start_time) > settings.time_limit) {
settings.log.printf("Barrier time limit exceeded\n");
return lp_status_t::TIME_LIMIT;
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }

// Compute the affine step. This is the call that (re)factorizes the
// augmented system, so the IR residual here drives the adaptation of
Expand All @@ -4295,10 +4282,7 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
status =
gpu_compute_search_direction(data, dual_perturb, primal_perturb, max_affine_residual);
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }

if (status < 0) {
return check_for_suboptimal_solution(data,
Expand All @@ -4313,14 +4297,7 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
relative_complementarity_residual,
solution);
}
if (toc(start_time) > settings.time_limit) {
settings.log.printf("Barrier time limit exceeded\n");
return lp_status_t::TIME_LIMIT;
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }

f_t mu_aff, sigma, new_mu;
compute_target_mu(data, mu, mu_aff, sigma, new_mu);
Expand All @@ -4336,10 +4313,7 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
status =
gpu_compute_search_direction(data, dual_perturb, primal_perturb, max_corrector_residual);
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }
if (status < 0) {
return check_for_suboptimal_solution(data,
start_time,
Expand All @@ -4355,14 +4329,7 @@ lp_status_t barrier_solver_t<i_t, f_t>::solve(f_t start_time, lp_solution_t<i_t,
}
data.has_factorization = false;
data.has_solve_info = false;
if (toc(start_time) > settings.time_limit) {
settings.log.printf("Barrier time limit exceeded\n");
return lp_status_t::TIME_LIMIT;
}
if (settings.concurrent_halt != nullptr && *settings.concurrent_halt == 1) {
settings.log.printf("Barrier solver halted\n");
return lp_status_t::CONCURRENT_LIMIT;
}
if (auto st = barrier_limit_status()) { return *st; }

compute_final_direction(data);
f_t step_primal, step_dual;
Expand Down
Loading
Loading