refactor: retry with backoff
This commit is contained in:
@@ -17,12 +17,23 @@ use crate::{
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
use anyhow::{Result, anyhow};
|
use anyhow::{Result, anyhow};
|
||||||
|
use backoff::backoff::Backoff;
|
||||||
|
use backoff::{Error as BackoffError, ExponentialBackoff};
|
||||||
use compact_str::CompactString;
|
use compact_str::CompactString;
|
||||||
use flexi_logger::DeferredNow;
|
use flexi_logger::DeferredNow;
|
||||||
use log::Level;
|
use log::Level;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
use std::{error::Error, fmt, path::PathBuf, sync::Arc, time::Duration};
|
use std::{
|
||||||
|
error::Error,
|
||||||
|
fmt,
|
||||||
|
path::PathBuf,
|
||||||
|
sync::{
|
||||||
|
Arc,
|
||||||
|
atomic::{AtomicUsize, Ordering},
|
||||||
|
},
|
||||||
|
time::{Duration, Instant},
|
||||||
|
};
|
||||||
use tauri_plugin_mihomo::Error as MihomoError;
|
use tauri_plugin_mihomo::Error as MihomoError;
|
||||||
use tauri_plugin_shell::ShellExt;
|
use tauri_plugin_shell::ShellExt;
|
||||||
use tokio::time::sleep;
|
use tokio::time::sleep;
|
||||||
@@ -956,71 +967,97 @@ impl CoreManager {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const MAX_ATTEMPTS: usize = 15;
|
let max_wait = Duration::from_millis(3000);
|
||||||
const WAIT_INTERVAL_MS: u64 = 200;
|
let mut backoff_strategy = ExponentialBackoff {
|
||||||
|
initial_interval: Duration::from_millis(200),
|
||||||
|
max_interval: Duration::from_millis(200),
|
||||||
|
max_elapsed_time: Some(max_wait),
|
||||||
|
multiplier: 1.0,
|
||||||
|
randomization_factor: 0.0,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
backoff_strategy.reset();
|
||||||
|
|
||||||
for attempt in 0..MAX_ATTEMPTS {
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
let mut manager = SERVICE_MANAGER.lock().await;
|
let attempts_for_retry = Arc::clone(&attempts);
|
||||||
|
|
||||||
if matches!(manager.current(), ServiceStatus::Ready) {
|
let operation = || {
|
||||||
if attempt > 0 {
|
let attempts = Arc::clone(&attempts_for_retry);
|
||||||
|
|
||||||
|
async move {
|
||||||
|
let attempt = attempts.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let mut manager = SERVICE_MANAGER.lock().await;
|
||||||
|
|
||||||
|
if matches!(manager.current(), ServiceStatus::Ready) {
|
||||||
|
if attempt > 0 {
|
||||||
|
logging!(
|
||||||
|
info,
|
||||||
|
Type::Core,
|
||||||
|
"Service became ready for TUN after {} attempt(s)",
|
||||||
|
attempt + 1
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
if attempt == 0 {
|
||||||
|
logging!(
|
||||||
|
info,
|
||||||
|
Type::Core,
|
||||||
|
"TUN mode enabled but service not ready; waiting for service availability"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
match manager.init().await {
|
||||||
|
Ok(_) => {
|
||||||
|
logging_error!(Type::Core, manager.refresh().await);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
logging!(
|
||||||
|
debug,
|
||||||
|
Type::Core,
|
||||||
|
"Service connection attempt {} failed while waiting for TUN: {}",
|
||||||
|
attempt + 1,
|
||||||
|
err
|
||||||
|
);
|
||||||
|
return Err(BackoffError::transient(err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if matches!(manager.current(), ServiceStatus::Ready) {
|
||||||
logging!(
|
logging!(
|
||||||
info,
|
info,
|
||||||
Type::Core,
|
Type::Core,
|
||||||
"Service became ready for TUN after {} attempt(s)",
|
"Service became ready for TUN after {} attempt(s)",
|
||||||
attempt
|
attempt + 1
|
||||||
);
|
);
|
||||||
|
return Ok(());
|
||||||
}
|
}
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if attempt == 0 {
|
|
||||||
logging!(
|
logging!(
|
||||||
info,
|
debug,
|
||||||
Type::Core,
|
Type::Core,
|
||||||
"TUN mode enabled but service not ready; waiting for service availability"
|
"Service not ready after attempt {}; retrying with backoff",
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
match manager.init().await {
|
|
||||||
Ok(_) => {
|
|
||||||
logging_error!(Type::Core, manager.refresh().await);
|
|
||||||
}
|
|
||||||
Err(err) => {
|
|
||||||
logging!(
|
|
||||||
debug,
|
|
||||||
Type::Core,
|
|
||||||
"Service connection attempt {} failed while waiting for TUN: {}",
|
|
||||||
attempt + 1,
|
|
||||||
err
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if matches!(manager.current(), ServiceStatus::Ready) {
|
|
||||||
logging!(
|
|
||||||
info,
|
|
||||||
Type::Core,
|
|
||||||
"Service became ready for TUN after {} attempt(s)",
|
|
||||||
attempt + 1
|
attempt + 1
|
||||||
);
|
);
|
||||||
return;
|
|
||||||
|
Err(BackoffError::transient(anyhow!("Service not ready yet")))
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
drop(manager);
|
let wait_started = Instant::now();
|
||||||
|
|
||||||
if attempt + 1 == MAX_ATTEMPTS {
|
if let Err(err) = backoff::future::retry(backoff_strategy, operation).await {
|
||||||
let total_wait_ms = (MAX_ATTEMPTS as u64) * WAIT_INTERVAL_MS;
|
let total_attempts = attempts.load(Ordering::Relaxed);
|
||||||
logging!(
|
let waited_ms = wait_started.elapsed().as_millis();
|
||||||
warn,
|
logging!(
|
||||||
Type::Core,
|
warn,
|
||||||
"Service still not ready after waiting approximately {} ms; falling back to sidecar mode",
|
Type::Core,
|
||||||
total_wait_ms
|
"Service still not ready after waiting approximately {} ms ({} attempt(s)); falling back to sidecar mode: {}",
|
||||||
);
|
waited_ms,
|
||||||
break;
|
total_attempts,
|
||||||
}
|
err
|
||||||
|
);
|
||||||
sleep(Duration::from_millis(WAIT_INTERVAL_MS)).await;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user