diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 00911370..10e974e2 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -44,3 +44,30 @@ jobs: - name: Check formatting working-directory: src-tauri run: cargo fmt --check + + failover-e2e: + name: failover E2E test + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.91.1 + + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: src-tauri + key: failover-e2e + + - name: Run failover E2E test + working-directory: src-tauri + run: | + sandbox_home="$(mktemp -d)" + export HOME="$sandbox_home" + export USERPROFILE="$sandbox_home" + export CC_SWITCH_CONFIG_DIR="$sandbox_home/.cc-switch" + cargo test --test proxy_claude_forwarder_alignment proxy_claude_auto_failover_uses_activated_queue_providers -- --exact --nocapture diff --git a/src-tauri/src/app_config.rs b/src-tauri/src/app_config.rs index 35e2fa51..46642f50 100644 --- a/src-tauri/src/app_config.rs +++ b/src-tauri/src/app_config.rs @@ -883,6 +883,7 @@ mod tests { _lock: crate::test_support::TestHomeSettingsLock, original_home: Option, original_userprofile: Option, + original_config_dir: Option, } impl TempHome { @@ -891,9 +892,11 @@ mod tests { let lock = crate::test_support::lock_test_home_and_settings(); let original_home = env::var_os("HOME"); let original_userprofile = env::var_os("USERPROFILE"); + let original_config_dir = env::var_os("CC_SWITCH_CONFIG_DIR"); env::set_var("HOME", dir.path()); env::set_var("USERPROFILE", dir.path()); + env::set_var("CC_SWITCH_CONFIG_DIR", dir.path().join(".cc-switch")); crate::test_support::set_test_home_override(Some(dir.path())); crate::settings::reload_test_settings(); @@ -902,6 +905,7 @@ mod tests { _lock: lock, original_home, original_userprofile, + original_config_dir, } } } @@ -918,6 +922,11 @@ mod tests { None => env::remove_var("USERPROFILE"), } + match &self.original_config_dir { + Some(value) => env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => env::remove_var("CC_SWITCH_CONFIG_DIR"), + } + crate::test_support::set_test_home_override( self.original_home.as_deref().map(Path::new), ); @@ -936,7 +945,9 @@ mod tests { fn seed_stale_test_home_with_gemini_override(home: &std::path::Path) { let stale_gemini_dir = home.join("custom-gemini"); let _lock = crate::test_support::lock_test_home_and_settings(); + let original_config_dir = env::var_os("CC_SWITCH_CONFIG_DIR"); + env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); crate::test_support::set_test_home_override(Some(home)); crate::settings::reload_test_settings(); @@ -944,6 +955,11 @@ mod tests { settings.gemini_config_dir = Some(stale_gemini_dir.to_string_lossy().into_owned()); settings.save().expect("save stale settings"); crate::settings::reload_test_settings(); + + match original_config_dir { + Some(value) => env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => env::remove_var("CC_SWITCH_CONFIG_DIR"), + } } #[test] diff --git a/src-tauri/src/cli/tui/app.rs b/src-tauri/src/cli/tui/app.rs index 10e79c82..d648332c 100644 --- a/src-tauri/src/cli/tui/app.rs +++ b/src-tauri/src/cli/tui/app.rs @@ -32,8 +32,8 @@ mod tests; mod types; pub(crate) use app_state::{ - Action, App, ConfigItem, LocalProxySettingsItem, ProxyVisualTransition, SettingsItem, - WebDavConfigItem, PROXY_HERO_TRANSITION_TICKS, + Action, App, ConfigItem, LocalProxySettingsItem, MoveDirection, ProxyVisualTransition, + SettingsItem, WebDavConfigItem, PROXY_HERO_TRANSITION_TICKS, }; pub use editor_state::{EditorKind, EditorMode, EditorState, EditorSubmit}; pub(crate) use helpers::*; @@ -42,6 +42,10 @@ pub use types::{ TextSubmit, TextViewAction, TextViewState, Toast, ToastKind, }; +pub(crate) fn supports_failover_controls(app_type: &AppType) -> bool { + matches!(app_type, AppType::Claude | AppType::Codex | AppType::Gemini) +} + const PROVIDER_NOTES_MAX_CHARS: usize = 120; #[cfg(unix)] diff --git a/src-tauri/src/cli/tui/app/app_state.rs b/src-tauri/src/cli/tui/app/app_state.rs index 3c49b236..42705b23 100644 --- a/src-tauri/src/cli/tui/app/app_state.rs +++ b/src-tauri/src/cli/tui/app/app_state.rs @@ -73,6 +73,14 @@ pub enum Action { ProviderStreamCheck { id: String, }, + ProviderSetFailoverQueue { + id: String, + enabled: bool, + }, + ProviderMoveFailoverQueue { + id: String, + direction: MoveDirection, + }, ProviderQuotaRefresh { id: String, }, @@ -175,6 +183,10 @@ pub enum Action { SetProxyListenPort { port: u16, }, + SetProxyAutoFailover { + app_type: AppType, + enabled: bool, + }, SetOpenClawConfigDir { path: Option, }, @@ -364,15 +376,23 @@ impl SettingsItem { pub enum LocalProxySettingsItem { ListenAddress, ListenPort, + AutoFailover, } impl LocalProxySettingsItem { - pub const ALL: [LocalProxySettingsItem; 2] = [ + pub const ALL: [LocalProxySettingsItem; 3] = [ LocalProxySettingsItem::ListenAddress, LocalProxySettingsItem::ListenPort, + LocalProxySettingsItem::AutoFailover, ]; } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MoveDirection { + Up, + Down, +} + #[derive(Debug, Clone)] pub enum WebDavConfigItem { Settings, diff --git a/src-tauri/src/cli/tui/app/content_config.rs b/src-tauri/src/cli/tui/app/content_config.rs index 8d04615b..eb90f527 100644 --- a/src-tauri/src/cli/tui/app/content_config.rs +++ b/src-tauri/src/cli/tui/app/content_config.rs @@ -767,39 +767,52 @@ impl App { self.settings_proxy_idx = (self.settings_proxy_idx + 1).min(items_len - 1); Action::None } - KeyCode::Enter => { - if data.proxy.running { - self.push_toast( - texts::tui_toast_proxy_settings_stop_before_edit(), - ToastKind::Info, - ); - return Action::None; + KeyCode::Enter => match LocalProxySettingsItem::ALL.get(self.settings_proxy_idx) { + Some(LocalProxySettingsItem::AutoFailover) => { + if !supports_failover_controls(&self.app_type) { + return Action::None; + } + Action::SetProxyAutoFailover { + app_type: self.app_type.clone(), + enabled: !data.proxy.auto_failover_enabled, + } } - - match LocalProxySettingsItem::ALL.get(self.settings_proxy_idx) { - Some(LocalProxySettingsItem::ListenAddress) => { - self.overlay = Overlay::TextInput(TextInputState { - title: texts::tui_settings_proxy_title().to_string(), - prompt: texts::tui_settings_proxy_listen_address_prompt().to_string(), - buffer: data.proxy.configured_listen_address.clone(), - submit: TextSubmit::SettingsProxyListenAddress, - secret: false, - }); - Action::None + Some(LocalProxySettingsItem::ListenAddress) => { + if data.proxy.running { + self.push_toast( + texts::tui_toast_proxy_settings_stop_before_edit(), + ToastKind::Info, + ); + return Action::None; } - Some(LocalProxySettingsItem::ListenPort) => { - self.overlay = Overlay::TextInput(TextInputState { - title: texts::tui_settings_proxy_title().to_string(), - prompt: texts::tui_settings_proxy_listen_port_prompt().to_string(), - buffer: data.proxy.configured_listen_port.to_string(), - submit: TextSubmit::SettingsProxyListenPort, - secret: false, - }); - Action::None + self.overlay = Overlay::TextInput(TextInputState { + title: texts::tui_settings_proxy_title().to_string(), + prompt: texts::tui_settings_proxy_listen_address_prompt().to_string(), + buffer: data.proxy.configured_listen_address.clone(), + submit: TextSubmit::SettingsProxyListenAddress, + secret: false, + }); + Action::None + } + Some(LocalProxySettingsItem::ListenPort) => { + if data.proxy.running { + self.push_toast( + texts::tui_toast_proxy_settings_stop_before_edit(), + ToastKind::Info, + ); + return Action::None; } - None => Action::None, + self.overlay = Overlay::TextInput(TextInputState { + title: texts::tui_settings_proxy_title().to_string(), + prompt: texts::tui_settings_proxy_listen_port_prompt().to_string(), + buffer: data.proxy.configured_listen_port.to_string(), + submit: TextSubmit::SettingsProxyListenPort, + secret: false, + }); + Action::None } - } + None => Action::None, + }, _ => Action::None, } } diff --git a/src-tauri/src/cli/tui/app/content_entities.rs b/src-tauri/src/cli/tui/app/content_entities.rs index c712d608..18247086 100644 --- a/src-tauri/src/cli/tui/app/content_entities.rs +++ b/src-tauri/src/cli/tui/app/content_entities.rs @@ -1,6 +1,46 @@ use super::*; impl App { + fn provider_switch_action(&mut self, row: &super::data::ProviderRow, data: &UiData) -> Action { + if supports_failover_controls(&self.app_type) && data.proxy.auto_failover_enabled { + self.push_toast( + crate::t!( + "Manage provider priority in the failover queue while automatic failover is enabled.", + "自动故障转移开启时，请在故障转移队列中管理供应商优先级。" + ), + ToastKind::Info, + ); + return Action::None; + } + + if matches!(self.app_type, AppType::OpenCode) { + if row.is_in_config { + return Action::ProviderRemoveFromConfig { id: row.id.clone() }; + } + + return Action::ProviderSwitch { id: row.id.clone() }; + } + if matches!(self.app_type, AppType::OpenClaw) { + if row.is_in_config { + if row.is_default_model { + self.push_toast( + texts::tui_toast_provider_cannot_remove_default_model(), + ToastKind::Warning, + ); + return Action::None; + } + return Action::ProviderRemoveFromConfig { id: row.id.clone() }; + } + + return Action::ProviderSwitch { id: row.id.clone() }; + } + if row.is_current { + self.push_toast(texts::tui_toast_provider_already_in_use(), ToastKind::Info); + return Action::None; + } + Action::ProviderSwitch { id: row.id.clone() } + } + pub(crate) fn on_providers_key(&mut self, key: KeyEvent, data: &UiData) -> Action { let visible = visible_providers(&self.app_type, &self.filter, data); match key.code { @@ -38,36 +78,11 @@ impl App { self.open_provider_edit_form(row); Action::None } - KeyCode::Char('s') => { + KeyCode::Char('s') | KeyCode::Char(' ') => { let Some(row) = visible.get(self.provider_idx) else { return Action::None; }; - if matches!(self.app_type, AppType::OpenCode) { - if row.is_in_config { - return Action::ProviderRemoveFromConfig { id: row.id.clone() }; - } - - return Action::ProviderSwitch { id: row.id.clone() }; - } - if matches!(self.app_type, AppType::OpenClaw) { - if row.is_in_config { - if row.is_default_model { - self.push_toast( - texts::tui_toast_provider_cannot_remove_default_model(), - ToastKind::Warning, - ); - return Action::None; - } - return Action::ProviderRemoveFromConfig { id: row.id.clone() }; - } - - return Action::ProviderSwitch { id: row.id.clone() }; - } - if row.is_current { - self.push_toast(texts::tui_toast_provider_already_in_use(), ToastKind::Info); - return Action::None; - } - Action::ProviderSwitch { id: row.id.clone() } + self.provider_switch_action(row, data) } KeyCode::Char('x') => { let Some(row) = visible.get(self.provider_idx) else { @@ -149,6 +164,22 @@ impl App { }; Action::ProviderStreamCheck { id: row.id.clone() } } + KeyCode::Char('f') => { + if !supports_failover_controls(&self.app_type) { + return Action::None; + } + let selected = visible + .get(self.provider_idx) + .and_then(|row| { + failover_queue_rows(data) + .iter() + .position(|provider_row| provider_row.id == row.id) + }) + .unwrap_or(0); + self.overlay = Overlay::FailoverQueueManager { selected }; + Action::None + } + KeyCode::Char('<') | KeyCode::Char('>') => Action::None, KeyCode::Char('r') => { let Some(row) = visible.get(self.provider_idx) else { return Action::None; @@ -179,34 +210,7 @@ impl App { Action::None } KeyCode::Enter => Action::None, - KeyCode::Char('s') => { - if matches!(self.app_type, AppType::OpenCode) { - if row.is_in_config { - return Action::ProviderRemoveFromConfig { id: row.id.clone() }; - } - - return Action::ProviderSwitch { id: row.id.clone() }; - } - if matches!(self.app_type, AppType::OpenClaw) { - if row.is_in_config { - if row.is_default_model { - self.push_toast( - texts::tui_toast_provider_cannot_remove_default_model(), - ToastKind::Warning, - ); - return Action::None; - } - return Action::ProviderRemoveFromConfig { id: row.id.clone() }; - } - - return Action::ProviderSwitch { id: row.id.clone() }; - } - if row.is_current { - self.push_toast(texts::tui_toast_provider_already_in_use(), ToastKind::Info); - return Action::None; - } - Action::ProviderSwitch { id: row.id.clone() } - } + KeyCode::Char('s') | KeyCode::Char(' ') => self.provider_switch_action(row, data), KeyCode::Char('x') => { if !matches!(self.app_type, AppType::OpenClaw) { return Action::None; @@ -254,6 +258,18 @@ impl App { }; Action::ProviderStreamCheck { id: row.id.clone() } } + KeyCode::Char('f') => { + if !supports_failover_controls(&self.app_type) { + return Action::None; + } + let selected = failover_queue_rows(data) + .iter() + .position(|provider_row| provider_row.id == row.id) + .unwrap_or(0); + self.overlay = Overlay::FailoverQueueManager { selected }; + Action::None + } + KeyCode::Char('<') | KeyCode::Char('>') => Action::None, KeyCode::Char('r') => { if data::quota_target_for_provider(&self.app_type, row).is_none() { self.push_toast(texts::tui_toast_quota_not_available(), ToastKind::Info); diff --git a/src-tauri/src/cli/tui/app/helpers.rs b/src-tauri/src/cli/tui/app/helpers.rs index 72e72a5f..f35ab4bc 100644 --- a/src-tauri/src/cli/tui/app/helpers.rs +++ b/src-tauri/src/cli/tui/app/helpers.rs @@ -940,6 +940,32 @@ pub(crate) fn visible_providers<'a>( .collect() } +pub(crate) fn failover_queue_rows(data: &UiData) -> Vec<&super::data::ProviderRow> { + let mut rows = data.providers.rows.iter().collect::>(); + rows.sort_by( + |a, b| match (a.provider.in_failover_queue, b.provider.in_failover_queue) { + (true, true) => match (a.provider.sort_index, b.provider.sort_index) { + (Some(a_idx), Some(b_idx)) => a_idx.cmp(&b_idx).then_with(|| a.id.cmp(&b.id)), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => a.id.cmp(&b.id), + }, + (true, false) => std::cmp::Ordering::Less, + (false, true) => std::cmp::Ordering::Greater, + (false, false) => a.id.cmp(&b.id), + }, + ); + rows +} + +pub(crate) fn failover_queue_position(data: &UiData, provider_id: &str) -> Option { + failover_queue_rows(data) + .into_iter() + .filter(|row| row.provider.in_failover_queue) + .position(|row| row.id == provider_id) + .map(|idx| idx + 1) +} + pub(crate) fn supports_provider_stream_check(app_type: &AppType) -> bool { !matches!(app_type, AppType::OpenClaw) } diff --git a/src-tauri/src/cli/tui/app/overlay_handlers/pickers.rs b/src-tauri/src/cli/tui/app/overlay_handlers/pickers.rs index a2269f73..b8ffab3d 100644 --- a/src-tauri/src/cli/tui/app/overlay_handlers/pickers.rs +++ b/src-tauri/src/cli/tui/app/overlay_handlers/pickers.rs @@ -39,6 +39,9 @@ impl App { if let Some(action) = self.handle_skills_import_picker_key(key) { return Some(action); } + if let Some(action) = self.handle_failover_queue_manager_key(key, data) { + return Some(action); + } None } @@ -741,4 +744,76 @@ impl App { _ => Action::None, }) } + + fn handle_failover_queue_manager_key( + &mut self, + key: KeyEvent, + data: &UiData, + ) -> Option { + let Overlay::FailoverQueueManager { selected } = &mut self.overlay else { + return None; + }; + + let rows = failover_queue_rows(data); + if rows.is_empty() { + return Some(match key.code { + KeyCode::Esc => { + self.overlay = Overlay::None; + Action::None + } + KeyCode::Char('f') => Action::SetProxyAutoFailover { + app_type: self.app_type.clone(), + enabled: !data.proxy.auto_failover_enabled, + }, + _ => Action::None, + }); + } + + *selected = (*selected).min(rows.len() - 1); + let selected_row = rows[*selected]; + + Some(match key.code { + KeyCode::Esc => { + self.overlay = Overlay::None; + Action::None + } + KeyCode::Up => { + *selected = selected.saturating_sub(1); + Action::None + } + KeyCode::Down => { + *selected = (*selected + 1).min(rows.len() - 1); + Action::None + } + KeyCode::Char('f') => Action::SetProxyAutoFailover { + app_type: self.app_type.clone(), + enabled: !data.proxy.auto_failover_enabled, + }, + KeyCode::Char(' ') | KeyCode::Enter => Action::ProviderSetFailoverQueue { + id: selected_row.id.clone(), + enabled: !selected_row.provider.in_failover_queue, + }, + KeyCode::Char('<') | KeyCode::Char('u') => { + if selected_row.provider.in_failover_queue { + Action::ProviderMoveFailoverQueue { + id: selected_row.id.clone(), + direction: MoveDirection::Up, + } + } else { + Action::None + } + } + KeyCode::Char('>') | KeyCode::Char('d') => { + if selected_row.provider.in_failover_queue { + Action::ProviderMoveFailoverQueue { + id: selected_row.id.clone(), + direction: MoveDirection::Down, + } + } else { + Action::None + } + } + _ => Action::None, + }) + } } diff --git a/src-tauri/src/cli/tui/app/tests.rs b/src-tauri/src/cli/tui/app/tests.rs index e938de90..a136075d 100644 --- a/src-tauri/src/cli/tui/app/tests.rs +++ b/src-tauri/src/cli/tui/app/tests.rs @@ -9209,6 +9209,390 @@ mod tests { assert_eq!(format, super::super::form::ClaudeApiFormat::OpenAiChat); } + fn failover_provider_row( + id: &str, + name: &str, + settings_config: serde_json::Value, + in_failover_queue: bool, + sort_index: Option, + ) -> ProviderRow { + let mut provider = + Provider::with_id(id.to_string(), name.to_string(), settings_config, None); + provider.in_failover_queue = in_failover_queue; + provider.sort_index = sort_index; + + ProviderRow { + id: id.to_string(), + provider, + api_url: Some("https://example.com".to_string()), + is_current: false, + is_in_config: true, + is_saved: true, + is_default_model: false, + primary_model_id: None, + default_model_id: None, + } + } + + #[test] + fn providers_space_switches_provider_when_failover_disabled() { + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.proxy.auto_failover_enabled = false; + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + false, + None, + )); + + let action = app.on_key(key(KeyCode::Char(' ')), &data); + assert!(matches!(action, Action::ProviderSwitch { id } if id == "p1")); + } + + #[test] + fn providers_space_is_blocked_when_failover_enabled() { + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.proxy.auto_failover_enabled = true; + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + true, + Some(0), + )); + + let action = app.on_key(key(KeyCode::Char(' ')), &data); + assert!(matches!(action, Action::None)); + assert!(matches!(app.toast.as_ref(), Some(toast) if toast.kind == ToastKind::Info)); + } + + #[test] + fn providers_s_key_is_blocked_when_failover_enabled() { + let mut app = App::new(Some(AppType::Codex)); + app.route = Route::Providers; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.proxy.auto_failover_enabled = true; + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"model_provider":{"base_url":"https://example.com"}}), + true, + Some(0), + )); + + let action = app.on_key(key(KeyCode::Char('s')), &data); + assert!(matches!(action, Action::None)); + assert!(matches!(app.toast.as_ref(), Some(toast) if toast.kind == ToastKind::Info)); + } + + #[test] + fn providers_move_keys_do_not_move_failover_queue() { + let mut app = App::new(Some(AppType::Gemini)); + app.route = Route::Providers; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"baseUrl":"https://example.com"}), + true, + Some(0), + )); + + assert!(matches!( + app.on_key(key(KeyCode::Char('<')), &data), + Action::None + )); + assert!(matches!( + app.on_key(key(KeyCode::Char('>')), &data), + Action::None + )); + } + + #[test] + fn provider_detail_move_keys_do_not_move_failover_queue() { + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::ProviderDetail { + id: "p1".to_string(), + }; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + true, + Some(0), + )); + + assert!(matches!( + app.on_key(key(KeyCode::Char('<')), &data), + Action::None + )); + assert!(matches!( + app.on_key(key(KeyCode::Char('>')), &data), + Action::None + )); + } + + #[test] + fn failover_queue_manager_f_toggles_auto_failover() { + let mut app = App::new(Some(AppType::Claude)); + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + + let mut data = UiData::default(); + data.proxy.auto_failover_enabled = true; + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + true, + Some(0), + )); + + let action = app.on_key(key(KeyCode::Char('f')), &data); + assert!(matches!( + action, + Action::SetProxyAutoFailover { app_type, enabled } + if app_type == AppType::Claude && !enabled + )); + } + + #[test] + fn failover_queue_manager_f_toggles_auto_failover_when_empty() { + let mut app = App::new(Some(AppType::Gemini)); + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + + let mut data = UiData::default(); + data.proxy.auto_failover_enabled = false; + + let action = app.on_key(key(KeyCode::Char('f')), &data); + assert!(matches!( + action, + Action::SetProxyAutoFailover { app_type, enabled } + if app_type == AppType::Gemini && enabled + )); + } + + #[test] + fn providers_f_key_opens_failover_queue_manager_for_supported_apps() { + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + false, + None, + )); + + let action = app.on_key(key(KeyCode::Char('f')), &data); + assert!(matches!(action, Action::None)); + assert!(matches!( + app.overlay, + Overlay::FailoverQueueManager { selected: 0 } + )); + } + + #[test] + fn provider_detail_f_key_opens_failover_queue_manager_for_supported_apps() { + let mut app = App::new(Some(AppType::Gemini)); + app.route = Route::ProviderDetail { + id: "p2".to_string(), + }; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"baseUrl":"https://example.com"}), + true, + Some(1), + )); + data.providers.rows.push(failover_provider_row( + "p2", + "Provider Two", + json!({"baseUrl":"https://example.com"}), + false, + None, + )); + + let action = app.on_key(key(KeyCode::Char('f')), &data); + assert!(matches!(action, Action::None)); + assert!(matches!( + app.overlay, + Overlay::FailoverQueueManager { selected: 1 } + )); + } + + #[test] + fn failover_queue_manager_space_toggles_selected_provider() { + let mut app = App::new(Some(AppType::Claude)); + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + false, + None, + )); + + let action = app.on_key(key(KeyCode::Char(' ')), &data); + assert!(matches!( + action, + Action::ProviderSetFailoverQueue { id, enabled } if id == "p1" && enabled + )); + } + + #[test] + fn failover_queue_manager_enter_removes_selected_queued_provider() { + let mut app = App::new(Some(AppType::Codex)); + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"model_provider":{"base_url":"https://example.com"}}), + true, + Some(1), + )); + + let action = app.on_key(key(KeyCode::Enter), &data); + assert!(matches!( + action, + Action::ProviderSetFailoverQueue { id, enabled } if id == "p1" && !enabled + )); + } + + #[test] + fn failover_queue_manager_move_keys_only_move_queued_provider() { + let mut app = App::new(Some(AppType::Codex)); + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"model_provider":{"base_url":"https://example.com"}}), + true, + Some(1), + )); + data.providers.rows.push(failover_provider_row( + "p2", + "Provider Two", + json!({"model_provider":{"base_url":"https://example.com"}}), + false, + None, + )); + + let action = app.on_key(key(KeyCode::Char('>')), &data); + assert!(matches!( + action, + Action::ProviderMoveFailoverQueue { + id, + direction: MoveDirection::Down, + } if id == "p1" + )); + + app.overlay = Overlay::FailoverQueueManager { selected: 1 }; + assert!(matches!( + app.on_key(key(KeyCode::Char('>')), &data), + Action::None + )); + } + + #[test] + fn failover_queue_manager_esc_closes_overlay() { + let mut app = App::new(Some(AppType::Claude)); + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + + let action = app.on_key(key(KeyCode::Esc), &UiData::default()); + assert!(matches!(action, Action::None)); + assert!(matches!(app.overlay, Overlay::None)); + } + + #[test] + fn unsupported_apps_ignore_failover_provider_keys() { + let mut app = App::new(Some(AppType::OpenCode)); + app.route = Route::Providers; + app.focus = Focus::Content; + + let mut data = UiData::default(); + data.providers.rows.push(failover_provider_row( + "p1", + "Provider One", + json!({"baseUrl":"https://example.com"}), + false, + None, + )); + + assert!(matches!( + app.on_key(key(KeyCode::Char('f')), &data), + Action::None + )); + assert!(matches!(app.overlay, Overlay::None)); + assert!(matches!( + app.on_key(key(KeyCode::Char('<')), &data), + Action::None + )); + } + + #[test] + fn settings_proxy_auto_failover_toggles_while_proxy_running() { + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::SettingsProxy; + app.focus = Focus::Content; + app.settings_proxy_idx = LocalProxySettingsItem::ALL + .iter() + .position(|item| *item == LocalProxySettingsItem::AutoFailover) + .expect("auto failover item should exist"); + + let mut data = UiData::default(); + data.proxy.running = true; + data.proxy.auto_failover_enabled = false; + + let action = app.on_key(key(KeyCode::Enter), &data); + assert!(matches!( + action, + Action::SetProxyAutoFailover { app_type, enabled } + if app_type == AppType::Claude && enabled + )); + } + + #[test] + fn unsupported_apps_ignore_settings_proxy_auto_failover() { + let mut app = App::new(Some(AppType::OpenClaw)); + app.route = Route::SettingsProxy; + app.focus = Focus::Content; + app.settings_proxy_idx = LocalProxySettingsItem::ALL + .iter() + .position(|item| *item == LocalProxySettingsItem::AutoFailover) + .expect("auto failover item should exist"); + + let action = app.on_key(key(KeyCode::Enter), &UiData::default()); + assert!(matches!(action, Action::None)); + } + #[test] fn openclaw_provider_edit_submit_uses_plain_edit_submit() { let mut app = App::new(Some(AppType::OpenClaw)); diff --git a/src-tauri/src/cli/tui/app/types.rs b/src-tauri/src/cli/tui/app/types.rs index c14b3f7c..907ba1c9 100644 --- a/src-tauri/src/cli/tui/app/types.rs +++ b/src-tauri/src/cli/tui/app/types.rs @@ -182,6 +182,9 @@ pub enum Overlay { CommonSnippetPicker { selected: usize, }, + FailoverQueueManager { + selected: usize, + }, CommonSnippetView { app_type: AppType, view: TextViewState, diff --git a/src-tauri/src/cli/tui/data.rs b/src-tauri/src/cli/tui/data.rs index 011b9815..dabd47c3 100644 --- a/src-tauri/src/cli/tui/data.rs +++ b/src-tauri/src/cli/tui/data.rs @@ -216,6 +216,7 @@ pub struct ProxySnapshot { pub enabled: bool, pub running: bool, pub managed_runtime: bool, + pub auto_failover_enabled: bool, pub claude_takeover: bool, pub codex_takeover: bool, pub gemini_takeover: bool, @@ -916,6 +917,7 @@ fn load_proxy_snapshot(app_type: &AppType) -> Result { runtime.block_on(async { let config = state.proxy_service.get_global_config().await?; + let app_proxy_config = state.db.get_proxy_config_for_app(app_type.as_str()).await?; let runtime_status = state.proxy_service.get_status().await; let takeover = state .proxy_service @@ -952,6 +954,7 @@ fn load_proxy_snapshot(app_type: &AppType) -> Result { enabled: config.proxy_enabled, running: runtime_status.running, managed_runtime: runtime_status.managed_session_token.is_some(), + auto_failover_enabled: app_proxy_config.auto_failover_enabled, claude_takeover: takeover.claude, codex_takeover: takeover.codex, gemini_takeover: takeover.gemini, @@ -1006,33 +1009,41 @@ mod tests { struct HomeGuard { old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl HomeGuard { fn set(home: &Path) -> Self { let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { old_home, old_userprofile, + old_config_dir, } } } impl Drop for HomeGuard { fn drop(&mut self) { - match self.old_home.take() { + match &self.old_home { Some(value) => std::env::set_var("HOME", value), None => std::env::remove_var("HOME"), } - match self.old_userprofile.take() { + match &self.old_userprofile { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } @@ -1080,6 +1091,36 @@ mod tests { } } + #[test] + #[serial] + fn load_proxy_snapshot_reads_app_auto_failover_state() { + let _guard = lock_test_home_and_settings(); + let temp = tempdir().expect("create tempdir"); + let _home = HomeGuard::set(temp.path()); + + let state = load_state().expect("load state"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create runtime"); + runtime.block_on(async { + let mut config = state + .db + .get_proxy_config_for_app("claude") + .await + .expect("read claude app proxy config"); + config.auto_failover_enabled = true; + state + .db + .update_proxy_config_for_app(config) + .await + .expect("persist claude app proxy config"); + }); + + let snapshot = load_proxy_snapshot(&AppType::Claude).expect("load proxy snapshot"); + assert!(snapshot.auto_failover_enabled); + } + #[test] fn quota_target_detects_official_claude_by_explicit_category() { let mut official = test_provider_row("official", "Claude Official", json!({"env": {}})); diff --git a/src-tauri/src/cli/tui/runtime_actions/editor.rs b/src-tauri/src/cli/tui/runtime_actions/editor.rs index ee0f7141..43b98165 100644 --- a/src-tauri/src/cli/tui/runtime_actions/editor.rs +++ b/src-tauri/src/cli/tui/runtime_actions/editor.rs @@ -819,6 +819,7 @@ mod tests { _lock: TestHomeSettingsLock, old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl EnvGuard { @@ -826,14 +827,17 @@ mod tests { let lock = lock_test_home_and_settings(); let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { _lock: lock, old_home, old_userprofile, + old_config_dir, } } } @@ -848,6 +852,10 @@ mod tests { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } diff --git a/src-tauri/src/cli/tui/runtime_actions/mod.rs b/src-tauri/src/cli/tui/runtime_actions/mod.rs index 0af0f0c6..0f01b3e9 100644 --- a/src-tauri/src/cli/tui/runtime_actions/mod.rs +++ b/src-tauri/src/cli/tui/runtime_actions/mod.rs @@ -219,6 +219,12 @@ pub(crate) fn handle_action( _ => Ok(()), }, Action::ProviderStreamCheck { id } => providers::stream_check(&mut ctx, id), + Action::ProviderSetFailoverQueue { id, enabled } => { + providers::set_failover_queue(&mut ctx, id, enabled) + } + Action::ProviderMoveFailoverQueue { id, direction } => { + providers::move_failover_queue(&mut ctx, id, direction) + } Action::ProviderQuotaRefresh { .. } => Ok(()), Action::ProviderModelFetch { base_url, @@ -297,6 +303,9 @@ pub(crate) fn handle_action( settings::set_proxy_listen_address(&mut ctx, address) } Action::SetProxyListenPort { port } => settings::set_proxy_listen_port(&mut ctx, port), + Action::SetProxyAutoFailover { app_type, enabled } => { + settings::set_proxy_auto_failover(&mut ctx, app_type, enabled) + } Action::SetOpenClawConfigDir { path } => settings::set_openclaw_config_dir(&mut ctx, path), Action::SetProxyTakeover { app_type, enabled } => { settings::set_proxy_takeover(&mut ctx, app_type, enabled) @@ -347,6 +356,7 @@ mod tests { _lock: TestHomeSettingsLock, old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl EnvGuard { @@ -354,14 +364,17 @@ mod tests { let lock = lock_test_home_and_settings(); let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { _lock: lock, old_home, old_userprofile, + old_config_dir, } } } @@ -376,6 +389,10 @@ mod tests { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } diff --git a/src-tauri/src/cli/tui/runtime_actions/providers.rs b/src-tauri/src/cli/tui/runtime_actions/providers.rs index b588b6ba..885dbe4e 100644 --- a/src-tauri/src/cli/tui/runtime_actions/providers.rs +++ b/src-tauri/src/cli/tui/runtime_actions/providers.rs @@ -3,6 +3,7 @@ use crate::cli::tui::form::ClaudeApiFormat; use crate::error::AppError; use crate::openclaw_config::OpenClawDefaultModel; use crate::proxy::providers::get_claude_api_format; +use crate::services::provider::ProviderSortUpdate; use crate::services::ProviderService; use serde_json::Value; @@ -12,6 +13,62 @@ use super::super::form::ProviderAddField; use super::super::runtime_systems::{next_model_fetch_request_id, ModelFetchReq, StreamCheckReq}; use super::RuntimeActionContext; +fn active_proxy_failover_queue_guard_message() -> &'static str { + crate::t!( + "At least one provider must remain in the failover queue while proxy failover is active.", + "代理故障转移激活时，故障转移队列中必须至少保留一个供应商。" + ) +} + +fn provider_is_last_active_failover_queue_entry( + ctx: &RuntimeActionContext<'_>, + provider_id: &str, +) -> Result { + if !crate::cli::tui::app::supports_failover_controls(&ctx.app.app_type) + || !ctx + .data + .proxy + .routes_current_app_through_proxy(&ctx.app.app_type) + .unwrap_or(false) + { + return Ok(false); + } + + let state = load_state()?; + let app_key = ctx.app.app_type.as_str(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| AppError::Message(format!("failed to create async runtime: {e}")))?; + let auto_failover_enabled = runtime + .block_on(async { state.db.get_proxy_config_for_app(app_key).await })? + .auto_failover_enabled; + if !auto_failover_enabled { + return Ok(false); + } + + let queue = state.db.get_failover_queue(app_key)?; + Ok(queue.len() == 1 + && queue + .first() + .is_some_and(|item| item.provider_id == provider_id)) +} + +fn guard_last_active_failover_queue_entry( + ctx: &mut RuntimeActionContext<'_>, + provider_id: &str, +) -> Result { + if provider_is_last_active_failover_queue_entry(ctx, provider_id)? { + ctx.app.push_toast( + active_proxy_failover_queue_guard_message(), + ToastKind::Warning, + ); + return Ok(true); + } + + Ok(false) +} + pub(super) fn switch(ctx: &mut RuntimeActionContext<'_>, id: String) -> Result<(), AppError> { do_switch(ctx, id) } @@ -122,7 +179,127 @@ fn provider_switch_proxy_notice_api_format( provider_requires_local_proxy(app_type, provider).filter(|_| !proxy_ready) } +pub(super) fn set_failover_queue( + ctx: &mut RuntimeActionContext<'_>, + id: String, + enabled: bool, +) -> Result<(), AppError> { + if !crate::cli::tui::app::supports_failover_controls(&ctx.app.app_type) { + return Ok(()); + } + if ctx.data.providers.rows.iter().all(|row| row.id != id) { + return Err(AppError::InvalidInput(format!("Provider not found: {id}"))); + } + + let state = load_state()?; + if enabled { + state + .db + .add_to_failover_queue(ctx.app.app_type.as_str(), &id)?; + } else { + if guard_last_active_failover_queue_entry(ctx, &id)? { + return Ok(()); + } + state + .db + .remove_from_failover_queue(ctx.app.app_type.as_str(), &id)?; + } + + *ctx.data = UiData::load(&ctx.app.app_type)?; + ctx.app.push_toast( + if enabled { + crate::t!( + "Provider added to the failover queue.", + "供应商已加入故障转移队列。" + ) + } else { + crate::t!( + "Provider removed from the failover queue.", + "供应商已移出故障转移队列。" + ) + }, + ToastKind::Success, + ); + Ok(()) +} + +pub(super) fn move_failover_queue( + ctx: &mut RuntimeActionContext<'_>, + id: String, + direction: crate::cli::tui::app::MoveDirection, +) -> Result<(), AppError> { + if !crate::cli::tui::app::supports_failover_controls(&ctx.app.app_type) { + return Ok(()); + } + + let mut queued = ctx + .data + .providers + .rows + .iter() + .filter(|row| row.provider.in_failover_queue) + .cloned() + .collect::>(); + queued.sort_by( + |a, b| match (a.provider.sort_index, b.provider.sort_index) { + (Some(a_idx), Some(b_idx)) => a_idx.cmp(&b_idx).then_with(|| a.id.cmp(&b.id)), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => a.id.cmp(&b.id), + }, + ); + + let Some(index) = queued.iter().position(|row| row.id == id) else { + ctx.app.push_toast( + crate::t!( + "Add this provider to the failover queue before moving it.", + "请先将该供应商加入故障转移队列再调整顺序。" + ), + ToastKind::Info, + ); + return Ok(()); + }; + + let target = match direction { + crate::cli::tui::app::MoveDirection::Up if index > 0 => index - 1, + crate::cli::tui::app::MoveDirection::Down if index + 1 < queued.len() => index + 1, + _ => { + ctx.app.push_toast( + crate::t!( + "Provider is already at the edge of the failover queue.", + "该供应商已在故障转移队列边界。" + ), + ToastKind::Info, + ); + return Ok(()); + } + }; + + queued.swap(index, target); + let updates = queued + .iter() + .enumerate() + .map(|(sort_index, row)| ProviderSortUpdate { + id: row.id.clone(), + sort_index, + }) + .collect::>(); + + let state = load_state()?; + ProviderService::update_sort_order(&state, ctx.app.app_type.clone(), updates)?; + *ctx.data = UiData::load(&ctx.app.app_type)?; + ctx.app.push_toast( + crate::t!("Failover queue order updated.", "故障转移队列顺序已更新。"), + ToastKind::Success, + ); + Ok(()) +} + pub(super) fn delete(ctx: &mut RuntimeActionContext<'_>, id: String) -> Result<(), AppError> { + if guard_last_active_failover_queue_entry(ctx, &id)? { + return Ok(()); + } + let state = load_state()?; ProviderService::delete(&state, ctx.app.app_type.clone(), &id)?; ctx.app @@ -361,7 +538,7 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::cli::tui::app::App; + use crate::cli::tui::app::{App, MoveDirection}; use crate::cli::tui::app::{ConfirmAction, ConfirmOverlay}; use crate::cli::tui::runtime_systems::RequestTracker; use crate::cli::tui::terminal::TuiTerminal; @@ -376,6 +553,7 @@ mod tests { _lock: TestHomeSettingsLock, old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl EnvGuard { @@ -383,14 +561,17 @@ mod tests { let lock = lock_test_home_and_settings(); let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { _lock: lock, old_home, old_userprofile, + old_config_dir, } } } @@ -405,6 +586,10 @@ mod tests { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } @@ -671,6 +856,384 @@ mod tests { }) } + fn claude_queue_provider(id: &str) -> Provider { + Provider::with_id( + id.to_string(), + format!("Provider {id}"), + json!({"env":{"ANTHROPIC_BASE_URL":format!("https://{id}.example.com")}}), + None, + ) + } + + struct RuntimeActionFixture { + terminal: TuiTerminal, + app: App, + data: UiData, + proxy_loading: RequestTracker, + webdav_loading: RequestTracker, + update_check: RequestTracker, + } + + impl RuntimeActionFixture { + fn new(app_type: AppType) -> Self { + Self { + terminal: TuiTerminal::new_for_test().expect("create terminal"), + app: App::new(Some(app_type.clone())), + data: UiData::load(&app_type).expect("load ui data"), + proxy_loading: RequestTracker::default(), + webdav_loading: RequestTracker::default(), + update_check: RequestTracker::default(), + } + } + + fn ctx(&mut self) -> RuntimeActionContext<'_> { + RuntimeActionContext { + terminal: &mut self.terminal, + app: &mut self.app, + data: &mut self.data, + speedtest_req_tx: None, + stream_check_req_tx: None, + skills_req_tx: None, + proxy_req_tx: None, + proxy_loading: &mut self.proxy_loading, + local_env_req_tx: None, + webdav_req_tx: None, + webdav_loading: &mut self.webdav_loading, + update_req_tx: None, + update_check: &mut self.update_check, + model_fetch_req_tx: None, + } + } + } + + #[test] + #[serial(home_settings)] + fn active_proxy_failover_rejects_removing_last_queued_provider() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p1")) + .expect("add provider"); + state + .db + .add_to_failover_queue("claude", "p1") + .expect("queue provider"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create runtime"); + runtime.block_on(async { + let mut config = state.db.get_proxy_config_for_app("claude").await.unwrap(); + config.auto_failover_enabled = true; + state.db.update_proxy_config_for_app(config).await.unwrap(); + }); + + let mut fixture = RuntimeActionFixture::new(AppType::Claude); + fixture.data.proxy.running = true; + fixture.data.proxy.claude_takeover = true; + fixture.data.proxy.auto_failover_enabled = true; + set_failover_queue(&mut fixture.ctx(), "p1".to_string(), false) + .expect("attempt queue removal"); + + assert!(state + .db + .is_in_failover_queue("claude", "p1") + .expect("read queue membership")); + assert!(matches!(fixture.app.toast, Some(toast) if toast.kind == ToastKind::Warning)); + } + + #[test] + #[serial(home_settings)] + fn stopped_proxy_failover_allows_removing_last_queued_provider() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p1")) + .expect("add provider"); + state + .db + .add_to_failover_queue("claude", "p1") + .expect("queue provider"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create runtime"); + runtime.block_on(async { + let mut config = state.db.get_proxy_config_for_app("claude").await.unwrap(); + config.auto_failover_enabled = true; + state.db.update_proxy_config_for_app(config).await.unwrap(); + }); + + let mut fixture = RuntimeActionFixture::new(AppType::Claude); + fixture.data.proxy.running = false; + fixture.data.proxy.claude_takeover = true; + fixture.data.proxy.auto_failover_enabled = true; + set_failover_queue(&mut fixture.ctx(), "p1".to_string(), false) + .expect("remove provider from stopped queue"); + + assert!(!state + .db + .is_in_failover_queue("claude", "p1") + .expect("read queue membership")); + } + + #[test] + #[serial(home_settings)] + fn active_proxy_failover_allows_removing_one_of_multiple_queued_providers() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p1")) + .expect("add first provider"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p2")) + .expect("add second provider"); + state + .db + .add_to_failover_queue("claude", "p1") + .expect("queue first provider"); + state + .db + .add_to_failover_queue("claude", "p2") + .expect("queue second provider"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create runtime"); + runtime.block_on(async { + let mut config = state.db.get_proxy_config_for_app("claude").await.unwrap(); + config.auto_failover_enabled = true; + state.db.update_proxy_config_for_app(config).await.unwrap(); + }); + + let mut fixture = RuntimeActionFixture::new(AppType::Claude); + fixture.data.proxy.running = true; + fixture.data.proxy.claude_takeover = true; + fixture.data.proxy.auto_failover_enabled = true; + set_failover_queue(&mut fixture.ctx(), "p1".to_string(), false) + .expect("remove one queued provider"); + + assert!(!state + .db + .is_in_failover_queue("claude", "p1") + .expect("read first queue membership")); + assert!(state + .db + .is_in_failover_queue("claude", "p2") + .expect("read second queue membership")); + } + + #[test] + #[serial(home_settings)] + fn active_proxy_failover_allows_reordering_queued_providers() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p1")) + .expect("add first provider"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p2")) + .expect("add second provider"); + state + .db + .add_to_failover_queue("claude", "p1") + .expect("queue first provider"); + state + .db + .add_to_failover_queue("claude", "p2") + .expect("queue second provider"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create runtime"); + runtime.block_on(async { + let mut config = state.db.get_proxy_config_for_app("claude").await.unwrap(); + config.auto_failover_enabled = true; + state.db.update_proxy_config_for_app(config).await.unwrap(); + }); + + let mut fixture = RuntimeActionFixture::new(AppType::Claude); + fixture.data.proxy.running = true; + fixture.data.proxy.claude_takeover = true; + fixture.data.proxy.auto_failover_enabled = true; + move_failover_queue(&mut fixture.ctx(), "p2".to_string(), MoveDirection::Up) + .expect("move queued provider up"); + + let queue = state.db.get_failover_queue("claude").expect("read queue"); + assert_eq!(queue[0].provider_id, "p2"); + assert_eq!(queue[1].provider_id, "p1"); + } + + #[test] + #[serial(home_settings)] + fn active_proxy_failover_rejects_deleting_last_queued_provider() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + ProviderService::add(&state, AppType::Claude, claude_queue_provider("p1")) + .expect("add provider"); + state + .db + .add_to_failover_queue("claude", "p1") + .expect("queue provider"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("create runtime"); + runtime.block_on(async { + let mut config = state.db.get_proxy_config_for_app("claude").await.unwrap(); + config.auto_failover_enabled = true; + state.db.update_proxy_config_for_app(config).await.unwrap(); + }); + + let mut fixture = RuntimeActionFixture::new(AppType::Claude); + fixture.data.proxy.running = true; + fixture.data.proxy.claude_takeover = true; + fixture.data.proxy.auto_failover_enabled = true; + delete(&mut fixture.ctx(), "p1".to_string()).expect("attempt delete provider"); + + assert!(state + .db + .get_provider_by_id("p1", "claude") + .expect("read provider") + .is_some()); + assert!(state + .db + .is_in_failover_queue("claude", "p1") + .expect("read queue membership")); + } + + #[test] + #[serial(home_settings)] + fn provider_failover_queue_toggle_updates_database() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + ProviderService::add( + &state, + AppType::Claude, + Provider::with_id( + "p1".to_string(), + "Provider One".to_string(), + json!({"env":{"ANTHROPIC_BASE_URL":"https://example.com"}}), + None, + ), + ) + .expect("add provider"); + + let mut terminal = TuiTerminal::new_for_test().expect("create terminal"); + let mut app = App::new(Some(AppType::Claude)); + let mut data = UiData::load(&AppType::Claude).expect("load data"); + let mut proxy_loading = RequestTracker::default(); + let mut webdav_loading = RequestTracker::default(); + let mut update_check = RequestTracker::default(); + let mut ctx = RuntimeActionContext { + terminal: &mut terminal, + app: &mut app, + data: &mut data, + speedtest_req_tx: None, + stream_check_req_tx: None, + skills_req_tx: None, + proxy_req_tx: None, + proxy_loading: &mut proxy_loading, + local_env_req_tx: None, + webdav_req_tx: None, + webdav_loading: &mut webdav_loading, + update_req_tx: None, + update_check: &mut update_check, + model_fetch_req_tx: None, + }; + + set_failover_queue(&mut ctx, "p1".to_string(), true).expect("enable failover queue"); + assert!(state + .db + .is_in_failover_queue("claude", "p1") + .expect("read failover queue membership")); + assert!(ctx + .data + .providers + .rows + .iter() + .any(|row| row.id == "p1" && row.provider.in_failover_queue)); + + set_failover_queue(&mut ctx, "p1".to_string(), false).expect("disable failover queue"); + assert!(!state + .db + .is_in_failover_queue("claude", "p1") + .expect("read failover queue membership")); + } + + #[test] + #[serial(home_settings)] + fn provider_failover_queue_move_updates_sort_order() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = EnvGuard::set_home(temp_home.path()); + + let state = load_state().expect("load state"); + let mut first = Provider::with_id( + "first".to_string(), + "First".to_string(), + json!({"env":{"ANTHROPIC_BASE_URL":"https://first.example.com"}}), + None, + ); + first.sort_index = Some(0); + let mut second = Provider::with_id( + "second".to_string(), + "Second".to_string(), + json!({"env":{"ANTHROPIC_BASE_URL":"https://second.example.com"}}), + None, + ); + second.sort_index = Some(1); + ProviderService::add(&state, AppType::Claude, first).expect("add first provider"); + ProviderService::add(&state, AppType::Claude, second).expect("add second provider"); + state + .db + .add_to_failover_queue("claude", "first") + .expect("queue first provider"); + state + .db + .add_to_failover_queue("claude", "second") + .expect("queue second provider"); + + let mut terminal = TuiTerminal::new_for_test().expect("create terminal"); + let mut app = App::new(Some(AppType::Claude)); + let mut data = UiData::load(&AppType::Claude).expect("load data"); + let mut proxy_loading = RequestTracker::default(); + let mut webdav_loading = RequestTracker::default(); + let mut update_check = RequestTracker::default(); + let mut ctx = RuntimeActionContext { + terminal: &mut terminal, + app: &mut app, + data: &mut data, + speedtest_req_tx: None, + stream_check_req_tx: None, + skills_req_tx: None, + proxy_req_tx: None, + proxy_loading: &mut proxy_loading, + local_env_req_tx: None, + webdav_req_tx: None, + webdav_loading: &mut webdav_loading, + update_req_tx: None, + update_check: &mut update_check, + model_fetch_req_tx: None, + }; + + move_failover_queue( + &mut ctx, + "second".to_string(), + crate::cli::tui::app::MoveDirection::Up, + ) + .expect("move second provider up"); + + let queue = state.db.get_failover_queue("claude").expect("read queue"); + assert_eq!(queue[0].provider_id, "second"); + assert_eq!(queue[1].provider_id, "first"); + } + #[test] #[serial(home_settings)] fn provider_switch_does_not_show_restart_toast_when_live_sync_succeeds() { diff --git a/src-tauri/src/cli/tui/runtime_actions/settings.rs b/src-tauri/src/cli/tui/runtime_actions/settings.rs index d88942b5..165b71d2 100644 --- a/src-tauri/src/cli/tui/runtime_actions/settings.rs +++ b/src-tauri/src/cli/tui/runtime_actions/settings.rs @@ -46,6 +46,45 @@ pub(super) fn set_proxy_listen_port( }) } +pub(super) fn set_proxy_auto_failover( + ctx: &mut RuntimeActionContext<'_>, + app_type: AppType, + enabled: bool, +) -> Result<(), AppError> { + let state = load_state()?; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| AppError::Message(format!("failed to create async runtime: {e}")))?; + + let queue_empty = state.db.get_failover_queue(app_type.as_str())?.is_empty(); + runtime.block_on(async { + let mut config = state.db.get_proxy_config_for_app(app_type.as_str()).await?; + config.auto_failover_enabled = enabled; + state.db.update_proxy_config_for_app(config).await + })?; + + *ctx.data = UiData::load(&ctx.app.app_type)?; + ctx.app.push_toast( + if enabled { + crate::t!("Automatic failover enabled.", "自动故障转移已开启。") + } else { + crate::t!("Automatic failover disabled.", "自动故障转移已关闭。") + }, + super::super::app::ToastKind::Success, + ); + if enabled && queue_empty { + ctx.app.push_toast( + crate::t!( + "Add providers to the failover queue before routing traffic through the proxy.", + "请先将供应商加入故障转移队列，再让流量经过代理。" + ), + super::super::app::ToastKind::Warning, + ); + } + Ok(()) +} + pub(super) fn set_openclaw_config_dir( ctx: &mut RuntimeActionContext<'_>, path: Option, diff --git a/src-tauri/src/cli/tui/ui/config.rs b/src-tauri/src/cli/tui/ui/config.rs index 04c8fe73..01a1d58f 100644 --- a/src-tauri/src/cli/tui/ui/config.rs +++ b/src-tauri/src/cli/tui/ui/config.rs @@ -23,6 +23,7 @@ pub(super) fn local_proxy_settings_item_label(item: &LocalProxySettingsItem) -> match item { LocalProxySettingsItem::ListenAddress => texts::tui_settings_proxy_listen_address_label(), LocalProxySettingsItem::ListenPort => texts::tui_settings_proxy_listen_port_label(), + LocalProxySettingsItem::AutoFailover => crate::t!("Automatic failover", "自动故障转移"), } } @@ -2500,6 +2501,14 @@ pub(super) fn render_settings_proxy( local_proxy_settings_item_label(item).to_string(), data.proxy.configured_listen_port.to_string(), ), + LocalProxySettingsItem::AutoFailover => ( + local_proxy_settings_item_label(item).to_string(), + if data.proxy.auto_failover_enabled { + texts::enabled().to_string() + } else { + texts::disabled().to_string() + }, + ), }) .collect::>(); @@ -2538,8 +2547,15 @@ pub(super) fn render_settings_proxy( ]) .split(inner); - if app.focus == Focus::Content && !data.proxy.running { - render_key_bar_center(frame, chunks[0], theme, &[("Enter", texts::tui_key_edit())]); + if app.focus == Focus::Content { + let key_label = match LocalProxySettingsItem::ALL.get(app.settings_proxy_idx) { + Some(LocalProxySettingsItem::AutoFailover) => texts::tui_key_toggle(), + _ if data.proxy.running => "", + _ => texts::tui_key_edit(), + }; + if !key_label.is_empty() { + render_key_bar_center(frame, chunks[0], theme, &[("Enter", key_label)]); + } } let table = Table::new( diff --git a/src-tauri/src/cli/tui/ui/overlay/pickers.rs b/src-tauri/src/cli/tui/ui/overlay/pickers.rs index 4a4e603b..ce4d479f 100644 --- a/src-tauri/src/cli/tui/ui/overlay/pickers.rs +++ b/src-tauri/src/cli/tui/ui/overlay/pickers.rs @@ -501,6 +501,134 @@ pub(super) fn render_openclaw_tools_profile_picker_overlay( frame.render_stateful_widget(list, body_area, &mut state); } +pub(super) fn render_failover_queue_manager_overlay( + frame: &mut Frame<'_>, + data: &UiData, + content_area: Rect, + theme: &theme::Theme, + selected: usize, +) { + let area = centered_rect_fixed(OVERLAY_FIXED_LG.0, 16, content_area); + frame.render_widget(Clear, area); + + let outer = Block::default() + .borders(Borders::ALL) + .border_type(BorderType::Plain) + .border_style(overlay_border_style(theme, false)) + .title(crate::t!("Failover Queue", "故障转移队列")); + frame.render_widget(outer.clone(), area); + let inner = outer.inner(area); + + let chunks = Layout::default() + .direction(Direction::Vertical) + .constraints([ + Constraint::Length(1), + Constraint::Length(1), + Constraint::Min(0), + Constraint::Length(2), + ]) + .split(inner); + + render_key_bar_center( + frame, + chunks[0], + theme, + &[ + ("↑↓", texts::tui_key_select()), + ("f", crate::t!("enable/disable", "启用/禁用")), + ("Space/Enter", texts::tui_key_toggle()), + ("/u/d", texts::tui_key_move()), + ("Esc", texts::tui_key_close()), + ], + ); + + let status = if data.proxy.auto_failover_enabled { + crate::t!("Automatic failover: enabled", "自动故障转移：已开启") + } else { + crate::t!("Automatic failover: disabled", "自动故障转移：已关闭") + }; + frame.render_widget( + Paragraph::new(status) + .style(Style::default().fg(theme.dim)) + .alignment(Alignment::Center), + chunks[1], + ); + + let body_area = inset_top(chunks[2], 1); + let rows = app::failover_queue_rows(data); + if rows.is_empty() { + frame.render_widget( + Paragraph::new(crate::t!("No providers configured.", "暂无提供商配置。")) + .style(Style::default().fg(theme.dim)) + .alignment(Alignment::Center), + body_area, + ); + } else { + let header = Row::new(vec![ + Cell::from(""), + Cell::from(crate::t!("Queue", "队列")), + Cell::from(texts::header_name()), + Cell::from(texts::tui_header_api_url()), + ]) + .style(Style::default().fg(theme.dim).add_modifier(Modifier::BOLD)); + + let table_rows = rows.iter().map(|row| { + let marker = if row.provider.in_failover_queue { + texts::tui_marker_active() + } else { + texts::tui_marker_inactive() + }; + let queue = app::failover_queue_position(data, &row.id) + .map(|position| format!("#{position}")) + .unwrap_or_else(|| "-".to_string()); + let api_url = row.api_url.as_deref().unwrap_or_else(|| texts::tui_na()); + + Row::new(vec![ + Cell::from(marker), + Cell::from(queue), + Cell::from(row.provider.name.as_str()), + Cell::from(api_url.to_string()), + ]) + }); + + let table = Table::new( + table_rows, + [ + Constraint::Length(2), + Constraint::Length(8), + Constraint::Percentage(35), + Constraint::Percentage(65), + ], + ) + .header(header) + .block(Block::default().borders(Borders::NONE)) + .row_highlight_style(selection_style(theme)) + .highlight_symbol(highlight_symbol(theme)); + + let mut state = TableState::default(); + state.select(Some(selected.min(rows.len().saturating_sub(1)))); + frame.render_stateful_widget(table, body_area, &mut state); + } + + frame.render_widget( + Paragraph::new(if data.proxy.auto_failover_enabled { + crate::t!( + "Auto failover uses only checked providers, in queue order.", + "自动故障转移仅按队列顺序使用已勾选的提供商。" + ) + } else { + crate::t!( + "Direct provider selection is used. Enable failover to route by queue priority.", + "当前使用直接供应商选择。开启故障转移后将按队列优先级路由。" + ) + }) + .style(Style::default().fg(theme.dim)) + .alignment(Alignment::Center) + .wrap(Wrap { trim: false }), + chunks[3], + ); +} + pub(super) fn render_mcp_apps_picker_overlay( frame: &mut Frame<'_>, content_area: Rect, diff --git a/src-tauri/src/cli/tui/ui/overlay/render.rs b/src-tauri/src/cli/tui/ui/overlay/render.rs index 9cfd1b29..02d04db9 100644 --- a/src-tauri/src/cli/tui/ui/overlay/render.rs +++ b/src-tauri/src/cli/tui/ui/overlay/render.rs @@ -38,6 +38,15 @@ pub(crate) fn render_overlay( *selected, ) } + Overlay::FailoverQueueManager { selected } => { + super::pickers::render_failover_queue_manager_overlay( + frame, + data, + content_area, + theme, + *selected, + ) + } Overlay::CommonSnippetView { view, .. } => { super::basic::render_common_snippet_view_overlay( frame, diff --git a/src-tauri/src/cli/tui/ui/providers.rs b/src-tauri/src/cli/tui/ui/providers.rs index 772ba46e..23431f0d 100644 --- a/src-tauri/src/cli/tui/ui/providers.rs +++ b/src-tauri/src/cli/tui/ui/providers.rs @@ -1,3 +1,4 @@ +use crate::cli::tui::app::failover_queue_position; use crate::cli::tui::data; use super::*; @@ -24,6 +25,12 @@ fn opencode_status_label(row: &ProviderRow) -> &'static str { } } +fn failover_queue_label(data: &UiData, provider_id: &str) -> String { + failover_queue_position(data, provider_id) + .map(|position| format!("#{position}")) + .unwrap_or_else(|| "-".to_string()) +} + pub(super) fn provider_rows_filtered<'a>(app: &App, data: &'a UiData) -> Vec<&'a ProviderRow> { let query = app.filter.query_lower(); data.providers @@ -129,8 +136,14 @@ pub(super) fn render_providers( keys.push(("a", texts::tui_key_add())); keys.push(("i", texts::tui_key_import())); } else { - keys.extend([("s", texts::tui_key_switch()), ("a", texts::tui_key_add())]); - keys.extend([("e", texts::tui_key_edit()), ("d", texts::tui_key_delete())]); + if !data.proxy.auto_failover_enabled { + keys.push(("Space", texts::tui_key_switch())); + } + keys.extend([ + ("a", texts::tui_key_add()), + ("e", texts::tui_key_edit()), + ("d", texts::tui_key_delete()), + ]); if selected_supports_quota { keys.push(("r", texts::tui_key_refresh())); } @@ -140,16 +153,23 @@ pub(super) fn render_providers( } keys.push(("c", texts::tui_key_stream_check())); } + if crate::cli::tui::app::supports_failover_controls(&app.app_type) { + keys.push(("f", crate::t!("manage failover", "管理故障转移"))); + } } render_key_bar_center(frame, chunks[0], theme, &keys); } - let header = Row::new(vec![ + let failover_supported = crate::cli::tui::app::supports_failover_controls(&app.app_type); + let mut header_cells = vec![ Cell::from(""), Cell::from(texts::header_name()), Cell::from(texts::tui_header_api_url()), - ]) - .style(header_style); + ]; + if failover_supported { + header_cells.push(Cell::from(crate::t!("Failover", "故障转移"))); + } + let header = Row::new(header_cells).style(header_style); let rows = visible.iter().enumerate().map(|(idx, row)| { let marker = if matches!(app.app_type, crate::app_config::AppType::OpenClaw) { @@ -166,6 +186,12 @@ pub(super) fn render_providers( } else { "" } + } else if failover_supported && data.proxy.auto_failover_enabled { + if row.provider.in_failover_queue { + texts::tui_marker_active() + } else { + texts::tui_marker_inactive() + } } else if row.is_current { texts::tui_marker_active() } else { @@ -173,20 +199,27 @@ pub(super) fn render_providers( }; let api = row.api_url.as_deref().unwrap_or(texts::tui_na()); let show_quota = row.is_current || idx == app.provider_idx; - Row::new(vec![ + let mut cells = vec![ Cell::from(marker), Cell::from(provider_name_with_quota_line( app, data, row, show_quota, theme, )), Cell::from(api), - ]) + ]; + if failover_supported { + cells.push(Cell::from(failover_queue_label(data, &row.id))); + } + Row::new(cells) }); - let constraints = vec![ + let mut constraints = vec![ Constraint::Length(2), - Constraint::Percentage(48), - Constraint::Percentage(52), + Constraint::Percentage(44), + Constraint::Percentage(46), ]; + if failover_supported { + constraints.push(Constraint::Length(10)); + } let table = Table::new(rows, constraints) .header(header) @@ -247,7 +280,14 @@ pub(super) fn render_provider_detail( ]; keys } else { - let keys = vec![("s", texts::tui_key_switch()), ("e", texts::tui_key_edit())]; + let keys = if data.proxy.auto_failover_enabled { + vec![("e", texts::tui_key_edit())] + } else { + vec![ + ("Space", texts::tui_key_switch()), + ("e", texts::tui_key_edit()), + ] + }; keys }; if data::quota_target_for_provider(&app.app_type, row).is_some() { @@ -269,6 +309,9 @@ pub(super) fn render_provider_detail( keys.push(("o", texts::tui_key_launch_temp())); } keys.push(("c", texts::tui_key_stream_check())); + if crate::cli::tui::app::supports_failover_controls(&app.app_type) { + keys.push(("f", crate::t!("manage failover", "管理故障转移"))); + } } render_key_bar_center(frame, chunks[0], theme, &keys); } @@ -401,6 +444,32 @@ pub(super) fn render_provider_detail( } } + if crate::cli::tui::app::supports_failover_controls(&app.app_type) { + lines.push(Line::raw("")); + lines.push(Line::from(vec![ + Span::styled( + crate::t!("Failover queue", "故障转移队列"), + Style::default().fg(theme.accent), + ), + Span::raw(": "), + Span::raw(failover_queue_position(data, &row.id).map_or_else( + || crate::t!("Not in queue", "未加入队列").to_string(), + |position| format!("{} #{position}", crate::t!("Queued", "已加入")), + )), + ])); + lines.push(Line::from(vec![ + Span::styled( + crate::t!("Queue ordering", "队列排序"), + Style::default().fg(theme.accent), + ), + Span::raw(": "), + Span::raw(crate::t!( + "Uses provider order; moving queue entries can also reorder providers.", + "使用供应商顺序；移动队列项也可能重排供应商。" + )), + ])); + } + lines.extend(quota_detail_lines(app, data, row, theme)); frame.render_widget( @@ -567,7 +636,7 @@ mod tests { app.route = Route::Providers; app.focus = Focus::Content; let data = current_official_claude_data(); - let all = all_text(&super::super::tests::render(&app, &data)); + let all = all_text(&super::super::tests::render_with_size(&app, &data, 180, 40)); assert!(!all.contains(texts::tui_header_quota()), "{all}"); assert!( @@ -610,7 +679,7 @@ mod tests { default_model_id: None, }, ); - let all = all_text(&super::super::tests::render(&app, &data)); + let all = all_text(&super::super::tests::render_with_size(&app, &data, 180, 40)); assert!( all.contains(&format!("r {}", texts::tui_key_refresh())), diff --git a/src-tauri/src/cli/tui/ui/tests.rs b/src-tauri/src/cli/tui/ui/tests.rs index 6446efa8..7bdb196f 100644 --- a/src-tauri/src/cli/tui/ui/tests.rs +++ b/src-tauri/src/cli/tui/ui/tests.rs @@ -254,6 +254,7 @@ pub(super) struct SettingsEnvGuard { _lock: TestHomeSettingsLock, old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl SettingsEnvGuard { @@ -261,14 +262,17 @@ impl SettingsEnvGuard { let lock = lock_test_home_and_settings(); let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { _lock: lock, old_home, old_userprofile, + old_config_dir, } } } @@ -283,6 +287,10 @@ impl Drop for SettingsEnvGuard { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } @@ -507,6 +515,30 @@ pub(super) fn minimal_data(_app_type: &AppType) -> UiData { } } +fn failover_provider_row( + id: &str, + name: &str, + is_current: bool, + in_failover_queue: bool, + sort_index: Option, +) -> ProviderRow { + let mut provider = Provider::with_id(id.to_string(), name.to_string(), json!({}), None); + provider.in_failover_queue = in_failover_queue; + provider.sort_index = sort_index; + + ProviderRow { + id: id.to_string(), + provider, + api_url: Some("https://example.com".to_string()), + is_current, + is_in_config: true, + is_saved: true, + is_default_model: false, + primary_model_id: Some("claude-sonnet-4".to_string()), + default_model_id: None, + } +} + fn openclaw_provider_row(id: &str, name: &str, models: &[(&str, &str)]) -> ProviderRow { let settings_config = json!({ "models": models @@ -6719,6 +6751,134 @@ fn openclaw_provider_list_key_bar_uses_additive_mode_actions() { assert!(!all.contains("s switch")); } +#[test] +fn failover_provider_list_key_bar_hides_move_hint_and_gates_switch_hint() { + let _lock = lock_env(); + let _no_color = EnvGuard::remove("NO_COLOR"); + + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + let mut data = minimal_data(&app.app_type); + + let disabled_text = all_text(&render_with_size(&app, &data, 180, 40)); + let disabled_keys = line_with(&disabled_text, "manage failover"); + assert!(disabled_keys.contains("Space"), "{disabled_keys}"); + assert!(!disabled_keys.contains(""), "{disabled_keys}"); + + data.proxy.auto_failover_enabled = true; + let enabled_text = all_text(&render_with_size(&app, &data, 180, 40)); + let enabled_keys = line_with(&enabled_text, "manage failover"); + assert!(!enabled_keys.contains("Space"), "{enabled_keys}"); + assert!(!enabled_keys.contains(""), "{enabled_keys}"); +} + +#[test] +fn failover_provider_list_marks_queue_entries_when_enabled() { + let _lock = lock_env(); + let _no_color = EnvGuard::remove("NO_COLOR"); + + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + let mut data = minimal_data(&app.app_type); + data.proxy.auto_failover_enabled = true; + data.providers.current_id = "current".to_string(); + data.providers.rows = vec![ + failover_provider_row("current", "Current Provider", true, false, None), + failover_provider_row("queued", "Queued Provider", false, true, Some(1)), + ]; + + let buf = render(&app, &data); + let current_line = (0..buf.area.height) + .map(|y| line_at(&buf, y)) + .find(|line| line.contains("Current Provider") && line.contains("https://example.com")) + .expect("current provider row rendered"); + let queued_line = (0..buf.area.height) + .map(|y| line_at(&buf, y)) + .find(|line| line.contains("Queued Provider") && line.contains("https://example.com")) + .expect("queued provider row rendered"); + + assert!( + !current_line.contains(texts::tui_marker_active()), + "{current_line}" + ); + assert!( + queued_line.contains(texts::tui_marker_active()), + "{queued_line}" + ); + assert!(queued_line.contains("#1"), "{queued_line}"); +} + +#[test] +fn failover_provider_list_uses_current_marker_when_disabled() { + let _lock = lock_env(); + let _no_color = EnvGuard::remove("NO_COLOR"); + + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + let mut data = minimal_data(&app.app_type); + data.proxy.auto_failover_enabled = false; + data.providers.current_id = "current".to_string(); + data.providers.rows = vec![ + failover_provider_row("current", "Current Provider", true, false, None), + failover_provider_row("queued", "Queued Provider", false, true, Some(1)), + ]; + + let buf = render(&app, &data); + let current_line = (0..buf.area.height) + .map(|y| line_at(&buf, y)) + .find(|line| line.contains("Current Provider") && line.contains("https://example.com")) + .expect("current provider row rendered"); + + assert!( + current_line.contains(texts::tui_marker_active()), + "{current_line}" + ); +} + +#[test] +fn failover_queue_overlay_renders_enabled_state_and_toggle_hint() { + let _lock = lock_env(); + let _no_color = EnvGuard::remove("NO_COLOR"); + + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + let mut data = minimal_data(&app.app_type); + data.proxy.auto_failover_enabled = true; + + let all = all_text(&render(&app, &data)); + + assert!(all.contains("Automatic failover: enabled"), "{all}"); + assert!(all.contains("f enable/disable"), "{all}"); + assert!( + all.contains("Auto failover uses only checked providers"), + "{all}" + ); +} + +#[test] +fn failover_queue_overlay_renders_disabled_state_and_toggle_hint() { + let _lock = lock_env(); + let _no_color = EnvGuard::remove("NO_COLOR"); + + let mut app = App::new(Some(AppType::Claude)); + app.route = Route::Providers; + app.focus = Focus::Content; + app.overlay = Overlay::FailoverQueueManager { selected: 0 }; + let mut data = minimal_data(&app.app_type); + data.proxy.auto_failover_enabled = false; + + let all = all_text(&render(&app, &data)); + + assert!(all.contains("Automatic failover: disabled"), "{all}"); + assert!(all.contains("f enable/disable"), "{all}"); + assert!(all.contains("Direct provider selection is used"), "{all}"); +} + #[test] fn opencode_provider_list_key_bar_uses_config_membership_actions() { let _lock = lock_env(); diff --git a/src-tauri/src/openclaw_config.rs b/src-tauri/src/openclaw_config.rs index 65852936..aa2ff310 100644 --- a/src-tauri/src/openclaw_config.rs +++ b/src-tauri/src/openclaw_config.rs @@ -958,19 +958,27 @@ mod tests { struct HomeGuard { old_home: Option, + old_userprofile: Option, + old_config_dir: Option, old_test_home: Option, } impl HomeGuard { fn set(home: &Path) -> Self { let old_home = std::env::var_os("HOME"); + let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); let old_test_home = std::env::var_os("CC_SWITCH_TEST_HOME"); std::env::set_var("HOME", home); + std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); std::env::set_var("CC_SWITCH_TEST_HOME", home); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { old_home, + old_userprofile, + old_config_dir, old_test_home, } } @@ -982,6 +990,14 @@ mod tests { Some(value) => std::env::set_var("HOME", value), None => std::env::remove_var("HOME"), } + match self.old_userprofile.take() { + Some(value) => std::env::set_var("USERPROFILE", value), + None => std::env::remove_var("USERPROFILE"), + } + match self.old_config_dir.take() { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } match self.old_test_home.take() { Some(value) => std::env::set_var("CC_SWITCH_TEST_HOME", value), None => std::env::remove_var("CC_SWITCH_TEST_HOME"), diff --git a/src-tauri/src/proxy/forwarder.rs b/src-tauri/src/proxy/forwarder.rs index f56317da..1761a29e 100644 --- a/src-tauri/src/proxy/forwarder.rs +++ b/src-tauri/src/proxy/forwarder.rs @@ -152,7 +152,7 @@ impl RequestForwarder { } let claude_error_path = matches!(app_type, AppType::Claude); - let bypass_circuit_breaker = options.bypass_circuit_breaker || providers.len() == 1; + let bypass_circuit_breaker = options.bypass_circuit_breaker; let mut last_error = None; let mut attempted_provider = false; let mut pending_upstream_response = None; @@ -389,7 +389,7 @@ impl RequestForwarder { } let claude_error_path = matches!(app_type, AppType::Claude); - let bypass_circuit_breaker = options.bypass_circuit_breaker || providers.len() == 1; + let bypass_circuit_breaker = options.bypass_circuit_breaker; let mut last_error = None; let mut attempted_provider = false; let mut pending_upstream_response = None; diff --git a/src-tauri/src/proxy/forwarder/tests/provider_failover.rs b/src-tauri/src/proxy/forwarder/tests/provider_failover.rs index ca2822ae..20b00ffc 100644 --- a/src-tauri/src/proxy/forwarder/tests/provider_failover.rs +++ b/src-tauri/src/proxy/forwarder/tests/provider_failover.rs @@ -72,7 +72,7 @@ async fn single_provider_bypasses_open_breaker() { } #[tokio::test] -async fn single_provider_bypasses_open_breaker_even_without_explicit_bypass_option() { +async fn single_provider_respects_open_breaker_without_explicit_bypass_option() { let (base_url, hits, server) = spawn_mock_upstream(StatusCode::OK, json!({"ok": true})).await; let provider = claude_provider("p1", &base_url, None); let (db, router) = test_router().await; @@ -101,7 +101,7 @@ async fn single_provider_bypasses_open_breaker_even_without_explicit_bypass_opti .expect("open breaker"); assert!(!router.allow_provider_request("p1", "claude").await.allowed); - let result = forwarder + let error = forwarder .forward_buffered_response( &AppType::Claude, "/v1/messages", @@ -116,16 +116,16 @@ async fn single_provider_bypasses_open_breaker_even_without_explicit_bypass_opti RectifierConfig::default(), ) .await - .expect("single provider request should succeed even without explicit bypass"); + .expect_err("single provider request should respect an open breaker"); - assert_eq!(result.provider.id, provider.id); - assert_eq!(hits.count.load(Ordering::SeqCst), 1); + assert!(matches!(error, ProxyError::NoAvailableProvider)); + assert_eq!(hits.count.load(Ordering::SeqCst), 0); server.abort(); } #[tokio::test] -async fn single_streaming_provider_bypasses_open_breaker_even_without_explicit_bypass_option() { +async fn single_streaming_provider_respects_open_breaker_without_explicit_bypass_option() { let (base_url, hits, bodies, server) = spawn_scripted_streaming_upstream(vec![( StatusCode::OK, ScriptedStreamingBody::Sse( @@ -170,7 +170,7 @@ async fn single_streaming_provider_bypasses_open_breaker_even_without_explicit_b }] }); - let result = forwarder + let error = forwarder .forward_response( &AppType::Claude, "/v1/messages", @@ -185,16 +185,11 @@ async fn single_streaming_provider_bypasses_open_breaker_even_without_explicit_b RectifierConfig::default(), ) .await - .expect("single provider streaming request should succeed even without explicit bypass"); + .expect_err("single provider streaming request should respect an open breaker"); - assert_eq!(result.provider.id, provider.id); - assert_eq!(result.response.status(), StatusCode::OK); - assert!(matches!( - &result.response, - StreamingResponse::Live(response) if is_sse_response(response) - )); - assert_eq!(hits.count.load(Ordering::SeqCst), 1); - assert_eq!(bodies.lock().await.len(), 1); + assert!(matches!(error, ProxyError::NoAvailableProvider)); + assert_eq!(hits.count.load(Ordering::SeqCst), 0); + assert_eq!(bodies.lock().await.len(), 0); server.abort(); } @@ -251,6 +246,204 @@ async fn claude_buffered_failover_uses_second_provider_and_per_provider_endpoint secondary_server.abort(); } +#[tokio::test] +async fn failover_enabled_single_queued_negative_provider_does_not_use_non_queued_healthy_provider() +{ + let (queued_url, queued_hits, queued_server) = spawn_mock_upstream( + StatusCode::INTERNAL_SERVER_ERROR, + json!({"error": {"message": "queued down"}}), + ) + .await; + let (healthy_url, healthy_hits, healthy_server) = + spawn_mock_upstream(StatusCode::OK, json!({"id": "resp_healthy", "ok": true})).await; + let queued_provider = claude_provider("queued", &queued_url, None); + let healthy_provider = claude_provider("healthy", &healthy_url, None); + let (db, router) = test_router().await; + let forwarder = RequestForwarder::new(router.clone()).expect("create forwarder"); + + db.save_provider("claude", &queued_provider) + .expect("save queued provider"); + db.save_provider("claude", &healthy_provider) + .expect("save healthy provider"); + db.set_current_provider("claude", &healthy_provider.id) + .expect("set non-queued current provider"); + db.add_to_failover_queue("claude", &queued_provider.id) + .expect("queue negative provider"); + let mut config = db + .get_proxy_config_for_app("claude") + .await + .expect("load proxy config"); + config.auto_failover_enabled = true; + db.update_proxy_config_for_app(config) + .await + .expect("enable failover"); + + let selected = router + .select_providers("claude") + .await + .expect("select queued providers"); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].id, queued_provider.id); + + let error = forwarder + .forward_buffered_response( + &AppType::Claude, + "/v1/messages", + claude_request_body(), + &HeaderMap::new(), + selected, + ForwardOptions { + max_retries: 0, + request_timeout: Some(Duration::from_secs(2)), + bypass_circuit_breaker: false, + }, + RectifierConfig::default(), + ) + .await + .expect_err( + "single queued negative provider should fail without using non-queued healthy provider", + ); + + assert!(matches!( + error, + ProxyError::UpstreamError { status: 500, .. } + )); + assert_eq!(queued_hits.count.load(Ordering::SeqCst), 1); + assert_eq!(healthy_hits.count.load(Ordering::SeqCst), 0); + + queued_server.abort(); + healthy_server.abort(); +} + +#[tokio::test] +async fn failover_enabled_multiple_queued_providers_transfer_by_queue_priority() { + let (primary_url, primary_hits, primary_server) = spawn_mock_upstream( + StatusCode::INTERNAL_SERVER_ERROR, + json!({"error": {"message": "primary down"}}), + ) + .await; + let (secondary_url, secondary_hits, secondary_server) = + spawn_mock_upstream(StatusCode::OK, json!({"id": "resp_secondary", "ok": true})).await; + let primary_provider = claude_provider("primary", &primary_url, None); + let secondary_provider = claude_provider("secondary", &secondary_url, None); + let (db, router) = test_router().await; + let forwarder = RequestForwarder::new(router.clone()).expect("create forwarder"); + + db.save_provider("claude", &primary_provider) + .expect("save primary provider"); + db.save_provider("claude", &secondary_provider) + .expect("save secondary provider"); + db.add_to_failover_queue("claude", &primary_provider.id) + .expect("queue primary provider"); + db.add_to_failover_queue("claude", &secondary_provider.id) + .expect("queue secondary provider"); + let mut config = db + .get_proxy_config_for_app("claude") + .await + .expect("load proxy config"); + config.auto_failover_enabled = true; + db.update_proxy_config_for_app(config) + .await + .expect("enable failover"); + + let selected = router + .select_providers("claude") + .await + .expect("select queued providers"); + assert_eq!(selected[0].id, primary_provider.id); + assert_eq!(selected[1].id, secondary_provider.id); + + let result = forwarder + .forward_buffered_response( + &AppType::Claude, + "/v1/messages", + claude_request_body(), + &HeaderMap::new(), + selected, + ForwardOptions { + max_retries: 0, + request_timeout: Some(Duration::from_secs(2)), + bypass_circuit_breaker: false, + }, + RectifierConfig::default(), + ) + .await + .expect("secondary queued provider should succeed after primary failure"); + + assert_eq!(result.provider.id, secondary_provider.id); + assert_eq!(primary_hits.count.load(Ordering::SeqCst), 1); + assert_eq!(secondary_hits.count.load(Ordering::SeqCst), 1); + + primary_server.abort(); + secondary_server.abort(); +} + +#[tokio::test] +async fn failover_enabled_all_queued_providers_unavailable_fails_after_attempting_queue() { + let (primary_url, primary_hits, primary_server) = spawn_mock_upstream( + StatusCode::INTERNAL_SERVER_ERROR, + json!({"error": {"message": "primary down"}}), + ) + .await; + let (secondary_url, secondary_hits, secondary_server) = spawn_mock_upstream( + StatusCode::BAD_GATEWAY, + json!({"error": {"message": "secondary down"}}), + ) + .await; + let primary_provider = claude_provider("primary", &primary_url, None); + let secondary_provider = claude_provider("secondary", &secondary_url, None); + let (db, router) = test_router().await; + let forwarder = RequestForwarder::new(router.clone()).expect("create forwarder"); + + db.save_provider("claude", &primary_provider) + .expect("save primary provider"); + db.save_provider("claude", &secondary_provider) + .expect("save secondary provider"); + db.add_to_failover_queue("claude", &primary_provider.id) + .expect("queue primary provider"); + db.add_to_failover_queue("claude", &secondary_provider.id) + .expect("queue secondary provider"); + let mut config = db + .get_proxy_config_for_app("claude") + .await + .expect("load proxy config"); + config.auto_failover_enabled = true; + db.update_proxy_config_for_app(config) + .await + .expect("enable failover"); + + let selected = router + .select_providers("claude") + .await + .expect("select queued providers"); + + let error = forwarder + .forward_buffered_response( + &AppType::Claude, + "/v1/messages", + claude_request_body(), + &HeaderMap::new(), + selected, + ForwardOptions { + max_retries: 0, + request_timeout: Some(Duration::from_secs(2)), + bypass_circuit_breaker: false, + }, + RectifierConfig::default(), + ) + .await + .expect_err("all queued negative providers should fail"); + + assert!(matches!( + error, + ProxyError::UpstreamError { status: 502, .. } + )); + assert_eq!(primary_hits.count.load(Ordering::SeqCst), 1); + assert_eq!(secondary_hits.count.load(Ordering::SeqCst), 1); + + primary_server.abort(); + secondary_server.abort(); +} #[tokio::test] async fn plain_buffered_400_fails_over_to_next_provider() { let (primary_url, primary_hits, primary_server) = spawn_mock_upstream( @@ -468,7 +661,7 @@ async fn plain_streaming_422_json_error_fails_over_to_next_provider() { } #[tokio::test] -async fn single_candidate_with_failover_enabled_still_bypasses_open_breaker() { +async fn single_candidate_with_failover_enabled_respects_open_breaker() { let (base_url, hits, server) = spawn_mock_upstream(StatusCode::OK, json!({"ok": true})).await; let provider = claude_provider("p1", &base_url, None); let (db, router) = test_router().await; @@ -496,7 +689,7 @@ async fn single_candidate_with_failover_enabled_still_bypasses_open_breaker() { .await .expect("open breaker"); - let result = forwarder + let error = forwarder .forward_buffered_response( &AppType::Claude, "/v1/messages", @@ -511,10 +704,10 @@ async fn single_candidate_with_failover_enabled_still_bypasses_open_breaker() { RectifierConfig::default(), ) .await - .expect("single candidate should still bypass breaker when it is the only provider"); + .expect_err("single failover candidate should respect an open breaker"); - assert_eq!(result.provider.id, provider.id); - assert_eq!(hits.count.load(Ordering::SeqCst), 1); + assert!(matches!(error, ProxyError::NoAvailableProvider)); + assert_eq!(hits.count.load(Ordering::SeqCst), 0); server.abort(); } diff --git a/src-tauri/src/proxy/handler_context.rs b/src-tauri/src/proxy/handler_context.rs index 193566ba..1e76fa50 100644 --- a/src-tauri/src/proxy/handler_context.rs +++ b/src-tauri/src/proxy/handler_context.rs @@ -139,6 +139,7 @@ mod tests { dir: TempDir, original_home: Option, original_userprofile: Option, + original_config_dir: Option, } impl TempHome { @@ -146,15 +147,18 @@ mod tests { let dir = TempDir::new().expect("create temp home"); let original_home = env::var("HOME").ok(); let original_userprofile = env::var("USERPROFILE").ok(); + let original_config_dir = env::var("CC_SWITCH_CONFIG_DIR").ok(); env::set_var("HOME", dir.path()); env::set_var("USERPROFILE", dir.path()); + env::set_var("CC_SWITCH_CONFIG_DIR", dir.path().join(".cc-switch")); crate::settings::reload_test_settings(); Self { dir, original_home, original_userprofile, + original_config_dir, } } } @@ -171,6 +175,11 @@ mod tests { None => env::remove_var("USERPROFILE"), } + match &self.original_config_dir { + Some(value) => env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => env::remove_var("CC_SWITCH_CONFIG_DIR"), + } + crate::settings::reload_test_settings(); } } @@ -204,7 +213,7 @@ mod tests { } #[tokio::test] - #[serial] + #[serial(home_settings)] async fn load_uses_current_provider_id_at_request_start() { let _home = TempHome::new(); let db = Arc::new(Database::memory().expect("create memory database")); @@ -241,7 +250,7 @@ mod tests { } #[tokio::test] - #[serial] + #[serial(home_settings)] async fn load_uses_effective_current_provider_from_settings_at_request_start() { let _home = TempHome::new(); let db = Arc::new(Database::memory().expect("create memory database")); @@ -279,7 +288,7 @@ mod tests { } #[tokio::test] - #[serial] + #[serial(home_settings)] async fn load_captures_current_provider_before_later_awaits() { let _home = TempHome::new(); let db = Arc::new(Database::memory().expect("create memory database")); diff --git a/src-tauri/src/proxy/provider_router/tests.rs b/src-tauri/src/proxy/provider_router/tests.rs index 19acbc83..450e2047 100644 --- a/src-tauri/src/proxy/provider_router/tests.rs +++ b/src-tauri/src/proxy/provider_router/tests.rs @@ -10,6 +10,7 @@ struct TempHome { dir: TempDir, original_home: Option, original_userprofile: Option, + original_config_dir: Option, } impl TempHome { @@ -17,15 +18,18 @@ impl TempHome { let dir = TempDir::new().expect("failed to create temp home"); let original_home = env::var("HOME").ok(); let original_userprofile = env::var("USERPROFILE").ok(); + let original_config_dir = env::var("CC_SWITCH_CONFIG_DIR").ok(); env::set_var("HOME", dir.path()); env::set_var("USERPROFILE", dir.path()); + env::set_var("CC_SWITCH_CONFIG_DIR", dir.path().join(".cc-switch")); crate::settings::reload_test_settings(); Self { dir, original_home, original_userprofile, + original_config_dir, } } } @@ -42,12 +46,17 @@ impl Drop for TempHome { None => env::remove_var("USERPROFILE"), } + match &self.original_config_dir { + Some(value) => env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => env::remove_var("CC_SWITCH_CONFIG_DIR"), + } + crate::settings::reload_test_settings(); } } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_provider_router_creation() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -58,7 +67,7 @@ async fn test_provider_router_creation() { } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_failover_disabled_uses_current_provider() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -79,7 +88,7 @@ async fn test_failover_disabled_uses_current_provider() { } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_failover_disabled_prefers_effective_current_provider_from_settings() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -100,7 +109,7 @@ async fn test_failover_disabled_prefers_effective_current_provider_from_settings } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_failover_disabled_reloads_settings_for_long_lived_router() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -129,7 +138,7 @@ async fn test_failover_disabled_reloads_settings_for_long_lived_router() { } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_failover_enabled_uses_queue_order_ignoring_current() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -160,7 +169,7 @@ async fn test_failover_enabled_uses_queue_order_ignoring_current() { } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_failover_enabled_without_queue_returns_no_providers_configured() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -189,7 +198,52 @@ async fn test_failover_enabled_without_queue_returns_no_providers_configured() { } #[tokio::test] -#[serial] +#[serial(home_settings)] +async fn test_failover_enabled_single_open_queued_provider_does_not_use_non_queued_current() { + let _home = TempHome::new(); + let db = Arc::new(Database::memory().unwrap()); + + db.update_circuit_breaker_config(&CircuitBreakerConfig { + failure_threshold: 1, + timeout_seconds: 3600, + ..Default::default() + }) + .await + .unwrap(); + + let queued = Provider::with_id("queued".to_string(), "Queued".to_string(), json!({}), None); + let current = Provider::with_id( + "current".to_string(), + "Current".to_string(), + json!({}), + None, + ); + + db.save_provider("claude", &queued).unwrap(); + db.save_provider("claude", ¤t).unwrap(); + db.set_current_provider("claude", "current").unwrap(); + db.add_to_failover_queue("claude", "queued").unwrap(); + + let mut config = db.get_proxy_config_for_app("claude").await.unwrap(); + config.auto_failover_enabled = true; + db.update_proxy_config_for_app(config).await.unwrap(); + + let router = ProviderRouter::new(db.clone()); + router + .record_result("queued", "claude", false, false, Some("fail".to_string())) + .await + .unwrap(); + + let error = router + .select_providers("claude") + .await + .expect_err("auto failover should not select a non-queued current provider"); + + assert!(matches!(error, ProxyError::AllProvidersCircuitOpen)); +} + +#[tokio::test] +#[serial(home_settings)] async fn test_select_providers_does_not_consume_half_open_permit() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -228,7 +282,7 @@ async fn test_select_providers_does_not_consume_half_open_permit() { } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_release_permit_neutral_frees_half_open_slot() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); @@ -273,7 +327,7 @@ async fn test_release_permit_neutral_frees_half_open_slot() { } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn test_record_result_uses_app_failure_threshold_for_health_updates() { let _home = TempHome::new(); let db = Arc::new(Database::memory().unwrap()); diff --git a/src-tauri/src/proxy/response_handler/tests.rs b/src-tauri/src/proxy/response_handler/tests.rs index c19618a3..3c891e8f 100644 --- a/src-tauri/src/proxy/response_handler/tests.rs +++ b/src-tauri/src/proxy/response_handler/tests.rs @@ -25,6 +25,7 @@ struct TempHome { dir: TempDir, original_home: Option, original_userprofile: Option, + original_config_dir: Option, } impl TempHome { @@ -32,15 +33,18 @@ impl TempHome { let dir = TempDir::new().expect("create temp home"); let original_home = env::var("HOME").ok(); let original_userprofile = env::var("USERPROFILE").ok(); + let original_config_dir = env::var("CC_SWITCH_CONFIG_DIR").ok(); env::set_var("HOME", dir.path()); env::set_var("USERPROFILE", dir.path()); + env::set_var("CC_SWITCH_CONFIG_DIR", dir.path().join(".cc-switch")); crate::settings::reload_test_settings(); Self { dir, original_home, original_userprofile, + original_config_dir, } } } @@ -57,6 +61,11 @@ impl Drop for TempHome { None => env::remove_var("USERPROFILE"), } + match &self.original_config_dir { + Some(value) => env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => env::remove_var("CC_SWITCH_CONFIG_DIR"), + } + crate::settings::reload_test_settings(); } } @@ -250,7 +259,7 @@ async fn buffered_success_streaming_responses_do_not_record_termination_error() } #[tokio::test] -#[serial] +#[serial(home_settings)] async fn streaming_success_syncs_failover_state_after_body_drains() { let _home = TempHome::new(); let db = Arc::new(Database::memory().expect("memory db")); diff --git a/src-tauri/src/proxy/server.rs b/src-tauri/src/proxy/server.rs index 1a041278..a54368d6 100644 --- a/src-tauri/src/proxy/server.rs +++ b/src-tauri/src/proxy/server.rs @@ -185,6 +185,7 @@ mod tests { dir: TempDir, original_home: Option, original_userprofile: Option, + original_config_dir: Option, } impl TempHome { @@ -192,15 +193,18 @@ mod tests { let dir = TempDir::new().expect("create temp home"); let original_home = env::var("HOME").ok(); let original_userprofile = env::var("USERPROFILE").ok(); + let original_config_dir = env::var("CC_SWITCH_CONFIG_DIR").ok(); env::set_var("HOME", dir.path()); env::set_var("USERPROFILE", dir.path()); + env::set_var("CC_SWITCH_CONFIG_DIR", dir.path().join(".cc-switch")); crate::settings::reload_test_settings(); Self { dir, original_home, original_userprofile, + original_config_dir, } } } @@ -217,6 +221,11 @@ mod tests { None => env::remove_var("USERPROFILE"), } + match &self.original_config_dir { + Some(value) => env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => env::remove_var("CC_SWITCH_CONFIG_DIR"), + } + crate::settings::reload_test_settings(); } } @@ -282,7 +291,7 @@ mod tests { } #[tokio::test] - #[serial] + #[serial(home_settings)] async fn sync_successful_provider_selection_updates_state_after_failover() { let _home = TempHome::new(); let db = Arc::new(Database::memory().expect("create memory database")); @@ -388,7 +397,7 @@ mod tests { } #[tokio::test] - #[serial] + #[serial(home_settings)] async fn sync_successful_provider_selection_skips_backup_update_when_takeover_disabled() { let _home = TempHome::new(); let db = Arc::new(Database::memory().expect("create memory database")); diff --git a/src-tauri/src/services/provider/tests.rs b/src-tauri/src/services/provider/tests.rs index 3b9d9613..92c74f3f 100644 --- a/src-tauri/src/services/provider/tests.rs +++ b/src-tauri/src/services/provider/tests.rs @@ -12,6 +12,7 @@ struct EnvGuard { _lock: TestHomeSettingsLock, old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl EnvGuard { @@ -19,14 +20,17 @@ impl EnvGuard { let lock = lock_test_home_and_settings(); let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { _lock: lock, old_home, old_userprofile, + old_config_dir, } } } @@ -41,6 +45,10 @@ impl Drop for EnvGuard { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } @@ -734,6 +742,17 @@ async fn switch_updates_running_proxy_takeover_target_without_restart() { let state = state_from_config(config); state.save().expect("persist config snapshot to db"); + let mut runtime_config = state + .db + .get_global_proxy_config() + .await + .expect("load global proxy config"); + runtime_config.listen_port = 0; + state + .db + .update_global_proxy_config(runtime_config) + .await + .expect("set ephemeral proxy port"); state .proxy_service diff --git a/src-tauri/src/services/proxy.rs b/src-tauri/src/services/proxy.rs index a9aa9e5c..ba059b5f 100644 --- a/src-tauri/src/services/proxy.rs +++ b/src-tauri/src/services/proxy.rs @@ -211,6 +211,7 @@ impl ProxyService { Self::ensure_managed_sessions_supported()?; let app_type = Self::takeover_app_from_str(app_type)?; + self.validate_app_proxy_activation(&app_type).await?; let current_status = self.get_status().await; if current_status.running { return Err( @@ -884,6 +885,54 @@ impl ProxyService { Ok(()) } + async fn validate_app_proxy_activation(&self, app_type: &AppType) -> Result<(), String> { + let app_key = app_type.as_str(); + let app_proxy = self + .db + .get_proxy_config_for_app(app_key) + .await + .map_err(|error| format!("load proxy config for {app_key} failed: {error}"))?; + + if app_proxy.auto_failover_enabled { + if self + .db + .get_failover_queue(app_key) + .map_err(|error| format!("load failover queue for {app_key} failed: {error}"))? + .is_empty() + { + return Err( + "cannot enable proxy because automatic failover is enabled and the failover queue is empty" + .to_string(), + ); + } + return Ok(()); + } + + let Some(provider_id) = + crate::settings::get_effective_current_provider(self.db.as_ref(), app_type).map_err( + |error| { + format!( + "load effective current provider for {} failed: {error}", + app_type.as_str() + ) + }, + )? + else { + return Err("cannot enable proxy because no active provider is selected".to_string()); + }; + + if self + .db + .get_provider_by_id(&provider_id, app_key) + .map_err(|error| format!("load provider {provider_id} for {app_key} failed: {error}"))? + .is_none() + { + return Err("cannot enable proxy because no active provider is selected".to_string()); + } + + Ok(()) + } + pub async fn save_live_backup_snapshot( &self, app_type: &str, @@ -912,6 +961,8 @@ impl ProxyService { } async fn enable_takeover_for_app_unlocked(&self, app_type: &AppType) -> Result<(), String> { + self.validate_app_proxy_activation(app_type).await?; + if !self.is_running().await { let config = self.get_config().await.map_err(|e| e.to_string())?; self.start_with_resolved_config_unlocked(config).await?; @@ -2024,6 +2075,7 @@ mod tests { _lock: crate::test_support::TestHomeSettingsLock, old_home: Option, old_userprofile: Option, + old_config_dir: Option, } impl TestHomeEnvGuard { @@ -2031,14 +2083,17 @@ mod tests { let lock = lock_test_home_and_settings(); let old_home = std::env::var_os("HOME"); let old_userprofile = std::env::var_os("USERPROFILE"); + let old_config_dir = std::env::var_os("CC_SWITCH_CONFIG_DIR"); std::env::set_var("HOME", home); std::env::set_var("USERPROFILE", home); + std::env::set_var("CC_SWITCH_CONFIG_DIR", home.join(".cc-switch")); set_test_home_override(Some(home)); crate::settings::reload_test_settings(); Self { _lock: lock, old_home, old_userprofile, + old_config_dir, } } } @@ -2053,11 +2108,222 @@ mod tests { Some(value) => std::env::set_var("USERPROFILE", value), None => std::env::remove_var("USERPROFILE"), } + match &self.old_config_dir { + Some(value) => std::env::set_var("CC_SWITCH_CONFIG_DIR", value), + None => std::env::remove_var("CC_SWITCH_CONFIG_DIR"), + } set_test_home_override(self.old_home.as_deref().map(Path::new)); crate::settings::reload_test_settings(); } } + #[tokio::test] + #[serial] + async fn takeover_activation_rejects_missing_current_provider_when_failover_disabled() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = TestHomeEnvGuard::set(temp_home.path()); + + let db = Arc::new(Database::memory().expect("create database")); + let service = ProxyService::new(db.clone()); + + let error = service + .set_takeover_for_app("claude", true) + .await + .expect_err("takeover should require an active provider when failover is disabled"); + + assert!( + error.contains("cannot enable proxy because no active provider is selected"), + "{error}" + ); + assert!( + !db.get_proxy_config_for_app("claude") + .await + .expect("load claude proxy config") + .enabled, + "failed validation should not enable takeover state" + ); + } + + #[tokio::test] + #[serial] + async fn takeover_activation_allows_current_provider_when_failover_disabled() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = TestHomeEnvGuard::set(temp_home.path()); + std::fs::create_dir_all( + get_claude_settings_path() + .parent() + .expect("claude settings parent dir"), + ) + .expect("create ~/.claude"); + write_json_file( + &get_claude_settings_path(), + &json!({ + "env": { + "ANTHROPIC_BASE_URL": "https://api.anthropic.com", + "ANTHROPIC_AUTH_TOKEN": "fresh-live-token" + } + }), + ) + .expect("seed claude live config"); + + let db = Arc::new(Database::memory().expect("create database")); + let service = ProxyService::new(db.clone()); + let provider = Provider::with_id( + "claude-provider".to_string(), + "Claude Provider".to_string(), + json!({ + "env": { + "ANTHROPIC_BASE_URL": "https://api.anthropic.com", + "ANTHROPIC_AUTH_TOKEN": "stale-provider-token" + } + }), + None, + ); + db.save_provider("claude", &provider) + .expect("save claude provider"); + db.set_current_provider("claude", &provider.id) + .expect("set current claude provider"); + let mut runtime_config = service.get_config().await.expect("get proxy config"); + runtime_config.listen_port = 0; + service + .update_config(&runtime_config) + .await + .expect("persist runtime config"); + + service + .set_takeover_for_app("claude", true) + .await + .expect("takeover should allow an active provider when failover is disabled"); + + assert!( + db.get_proxy_config_for_app("claude") + .await + .expect("load claude proxy config") + .enabled + ); + service.stop().await.expect("stop proxy runtime"); + } + + #[tokio::test] + #[serial] + async fn takeover_activation_rejects_empty_queue_when_failover_enabled() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = TestHomeEnvGuard::set(temp_home.path()); + + let db = Arc::new(Database::memory().expect("create database")); + let service = ProxyService::new(db.clone()); + let provider = Provider::with_id( + "claude-provider".to_string(), + "Claude Provider".to_string(), + json!({ + "env": { + "ANTHROPIC_BASE_URL": "https://api.anthropic.com", + "ANTHROPIC_AUTH_TOKEN": "token" + } + }), + None, + ); + db.save_provider("claude", &provider) + .expect("save claude provider"); + db.set_current_provider("claude", &provider.id) + .expect("set current claude provider"); + let mut app_proxy = db + .get_proxy_config_for_app("claude") + .await + .expect("load claude proxy config"); + app_proxy.auto_failover_enabled = true; + db.update_proxy_config_for_app(app_proxy) + .await + .expect("enable auto failover"); + + let error = service + .set_takeover_for_app("claude", true) + .await + .expect_err("takeover should require a non-empty failover queue"); + + assert!( + error.contains( + "cannot enable proxy because automatic failover is enabled and the failover queue is empty" + ), + "{error}" + ); + assert!( + !db.get_proxy_config_for_app("claude") + .await + .expect("load claude proxy config") + .enabled, + "failed validation should not enable takeover state" + ); + } + + #[tokio::test] + #[serial] + async fn takeover_activation_allows_non_empty_queue_when_failover_enabled() { + let temp_home = TempDir::new().expect("create temp home"); + let _env = TestHomeEnvGuard::set(temp_home.path()); + std::fs::create_dir_all( + get_claude_settings_path() + .parent() + .expect("claude settings parent dir"), + ) + .expect("create ~/.claude"); + write_json_file( + &get_claude_settings_path(), + &json!({ + "env": { + "ANTHROPIC_BASE_URL": "https://api.anthropic.com", + "ANTHROPIC_AUTH_TOKEN": "fresh-live-token" + } + }), + ) + .expect("seed claude live config"); + + let db = Arc::new(Database::memory().expect("create database")); + let service = ProxyService::new(db.clone()); + let provider = Provider::with_id( + "claude-provider".to_string(), + "Claude Provider".to_string(), + json!({ + "env": { + "ANTHROPIC_BASE_URL": "https://api.anthropic.com", + "ANTHROPIC_AUTH_TOKEN": "stale-provider-token" + } + }), + None, + ); + db.save_provider("claude", &provider) + .expect("save claude provider"); + db.add_to_failover_queue("claude", &provider.id) + .expect("queue claude provider"); + let mut app_proxy = db + .get_proxy_config_for_app("claude") + .await + .expect("load claude proxy config"); + app_proxy.auto_failover_enabled = true; + db.update_proxy_config_for_app(app_proxy) + .await + .expect("enable auto failover"); + let mut runtime_config = service.get_config().await.expect("get proxy config"); + runtime_config.listen_port = 0; + service + .update_config(&runtime_config) + .await + .expect("persist runtime config"); + + service + .set_takeover_for_app("claude", true) + .await + .expect("takeover should allow a non-empty failover queue"); + + assert!( + db.get_proxy_config_for_app("claude") + .await + .expect("load claude proxy config") + .enabled + ); + service.stop().await.expect("stop proxy runtime"); + } + #[tokio::test] #[serial] async fn recover_takeovers_on_startup_cleans_claude_placeholder_only_residue_without_backup() { @@ -2718,6 +2984,15 @@ base_url = "https://api.openai.com/v1" #[tokio::test] #[serial] async fn managed_session_ready_info_accepts_persisted_session_without_status_probe() { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind unused proxy status port"); + let port = listener + .local_addr() + .expect("read unused proxy status port") + .port(); + drop(listener); + let db = Arc::new(Database::memory().expect("create database")); let service = ProxyService::new(db.clone()); @@ -2726,7 +3001,7 @@ base_url = "https://api.openai.com/v1" &serde_json::to_string(&PersistedProxyRuntimeSession { pid: 4242, address: "127.0.0.1".to_string(), - port: 15721, + port, started_at: "2026-03-10T00:00:00Z".to_string(), kind: PersistedProxyRuntimeSessionKind::ManagedExternal, session_token: Some("expected-session-token".to_string()), @@ -2741,7 +3016,7 @@ base_url = "https://api.openai.com/v1" .expect("persisted managed runtime marker should be treated as ready"); assert_eq!(info.address, "127.0.0.1"); - assert_eq!(info.port, 15721); + assert_eq!(info.port, port); assert_eq!(info.started_at, "2026-03-10T00:00:00Z"); } diff --git a/src-tauri/tests/proxy_claude_forwarder_alignment.rs b/src-tauri/tests/proxy_claude_forwarder_alignment.rs index c74e416a..b12c4a49 100644 --- a/src-tauri/tests/proxy_claude_forwarder_alignment.rs +++ b/src-tauri/tests/proxy_claude_forwarder_alignment.rs @@ -43,6 +43,11 @@ async fn bind_test_listener() -> tokio::net::TcpListener { ); } +#[derive(Clone, Default)] +struct CountingUpstreamState { + attempts: Arc, +} + #[derive(Clone, Default)] struct UpstreamState { request_body: Arc>>, @@ -205,6 +210,43 @@ async fn handle_anthropic_messages( ) } +async fn handle_failing_anthropic_messages( + State(state): State, + Json(_body): Json, +) -> impl IntoResponse { + state.attempts.fetch_add(1, Ordering::SeqCst); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(json!({"error": {"message": "primary unavailable"}})), + ) +} + +async fn handle_successful_anthropic_messages( + State(state): State, + Json(body): Json, +) -> impl IntoResponse { + state.attempts.fetch_add(1, Ordering::SeqCst); + ( + StatusCode::OK, + Json(json!({ + "id": "msg_failover_success", + "type": "message", + "role": "assistant", + "content": [{ + "type": "text", + "text": "failover ok" + }], + "model": body.get("model").cloned().unwrap_or_else(|| json!("")), + "stop_reason": "end_turn", + "stop_sequence": null, + "usage": { + "input_tokens": 1, + "output_tokens": 1 + } + })), + ) +} + async fn handle_scripted_anthropic_messages( State(state): State, Json(body): Json, @@ -412,6 +454,166 @@ async fn send_claude_request(service: &ProxyService, body: &Value) -> reqwest::R .expect("send request to proxy") } +#[tokio::test] +#[serial] +async fn proxy_claude_auto_failover_uses_activated_queue_providers() { + let _guard = lock_test_mutex(); + reset_test_fs(); + let _home = ensure_test_home(); + + let primary_state = CountingUpstreamState::default(); + let primary_listener = bind_test_listener().await; + let primary_addr = primary_listener + .local_addr() + .expect("read primary upstream address"); + let primary_state_for_server = primary_state.clone(); + let primary_handle = tokio::spawn(async move { + let _ = axum::serve( + primary_listener, + Router::new() + .route("/v1/messages", post(handle_failing_anthropic_messages)) + .with_state(primary_state_for_server), + ) + .await; + }); + + let secondary_state = CountingUpstreamState::default(); + let secondary_listener = bind_test_listener().await; + let secondary_addr = secondary_listener + .local_addr() + .expect("read secondary upstream address"); + let secondary_state_for_server = secondary_state.clone(); + let secondary_handle = tokio::spawn(async move { + let _ = axum::serve( + secondary_listener, + Router::new() + .route("/v1/messages", post(handle_successful_anthropic_messages)) + .with_state(secondary_state_for_server), + ) + .await; + }); + + let db = Arc::new(Database::memory().expect("create memory database")); + let primary_provider = Provider { + id: "primary".to_string(), + name: "Primary".to_string(), + settings_config: json!({ + "env": { + "ANTHROPIC_BASE_URL": format!("http://{}", primary_addr), + "ANTHROPIC_API_KEY": "sk-test-primary" + } + }), + website_url: None, + category: Some("claude".to_string()), + created_at: None, + sort_index: Some(0), + notes: None, + meta: None, + icon: None, + icon_color: None, + in_failover_queue: false, + }; + let secondary_provider = Provider { + id: "secondary".to_string(), + name: "Secondary".to_string(), + settings_config: json!({ + "env": { + "ANTHROPIC_BASE_URL": format!("http://{}", secondary_addr), + "ANTHROPIC_API_KEY": "sk-test-secondary" + } + }), + website_url: None, + category: Some("claude".to_string()), + created_at: None, + sort_index: Some(1), + notes: None, + meta: None, + icon: None, + icon_color: None, + in_failover_queue: false, + }; + + db.save_provider("claude", &primary_provider) + .expect("save primary provider"); + db.save_provider("claude", &secondary_provider) + .expect("save secondary provider"); + db.set_current_provider("claude", "primary") + .expect("set current provider"); + db.add_to_failover_queue("claude", "primary") + .expect("activate primary failover queue entry"); + db.add_to_failover_queue("claude", "secondary") + .expect("activate secondary failover queue entry"); + + let queue_before = db + .get_failover_queue("claude") + .expect("read queue before request"); + assert_eq!(queue_before[0].provider_id, "primary"); + assert_eq!(queue_before[1].provider_id, "secondary"); + + let mut app_proxy = db + .get_proxy_config_for_app("claude") + .await + .expect("read claude app proxy config"); + app_proxy.auto_failover_enabled = true; + db.update_proxy_config_for_app(app_proxy) + .await + .expect("enable auto failover"); + + let service = ProxyService::new(db.clone()); + let mut config = service.get_config().await.expect("read proxy config"); + config.listen_port = 0; + service + .update_config(&config) + .await + .expect("update proxy config"); + service.start().await.expect("start proxy service"); + + let response = send_claude_request( + &service, + &json!({ + "model": "claude-3-7-sonnet-20250219", + "max_tokens": 64, + "messages": [{ + "role": "user", + "content": [{ + "type": "text", + "text": "hello" + }] + }] + }), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let body: Value = response.json().await.expect("read failover response body"); + assert_eq!(body["content"][0]["text"], json!("failover ok")); + + assert_eq!(primary_state.attempts.load(Ordering::SeqCst), 1); + assert_eq!(secondary_state.attempts.load(Ordering::SeqCst), 1); + assert_eq!( + db.get_current_provider("claude") + .expect("read current provider after request") + .as_deref(), + Some("secondary") + ); + + let status = service.get_status().await; + assert_eq!(status.current_provider_id.as_deref(), Some("secondary")); + assert_eq!(status.current_provider.as_deref(), Some("Secondary")); + assert_eq!(status.failover_count, 1); + assert_eq!(status.active_targets.len(), 1); + assert_eq!(status.active_targets[0].provider_id, "secondary"); + + let queue_after = db + .get_failover_queue("claude") + .expect("read queue after request"); + assert_eq!(queue_after[0].provider_id, "primary"); + assert_eq!(queue_after[1].provider_id, "secondary"); + + service.stop().await.expect("stop proxy service"); + primary_handle.abort(); + secondary_handle.abort(); +} + #[tokio::test] #[serial] async fn proxy_claude_successful_failover_syncs_current_provider_and_status() {