feat(shipote): throughput card + rate-limit + snapshot incremental (fase Q)
- shipote-shell Flow channels card extiende con bytes_total + bytes/s por socket. Lookup helper evita borrows en closures. - DiscernPolicy.max_bytes_per_sec: splitter task hace sleep proporcional al tamaño de chunk tras cada broadcast. Token-bucket simple v1. - WorkspaceManager.dirty: AtomicBool. mark_dirty() en mutaciones que afectan al snapshot. save_snapshot skip si clean y path existe. restore_snapshot resetea dirty=false (hidratación no es mutation). 85 tests pasan (ente-incarnate 16, nouser-core 27, shipote-card 8, shipote-core 26, shipote-discern 5, yahweh-provider-fs 3). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -132,6 +132,7 @@ pub async fn run_pipeline(
|
||||
edges: edge_meta,
|
||||
tap,
|
||||
sample_bytes: spec.discern.sample_bytes,
|
||||
max_bytes_per_sec: spec.discern.max_bytes_per_sec,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -308,6 +309,9 @@ struct SplitterSpec {
|
||||
edges: Vec<EdgeMeta>,
|
||||
tap: bool,
|
||||
sample_bytes: usize,
|
||||
/// Rate-limit en bytes/s (0 = sin limit). Tras cada chunk de `n`
|
||||
/// bytes, splitter sleeps `n / max_bytes_per_sec` segundos.
|
||||
max_bytes_per_sec: u64,
|
||||
}
|
||||
|
||||
struct SplitterHandle {
|
||||
@@ -430,6 +434,7 @@ fn spawn_splitter(
|
||||
}
|
||||
broadcast_chunk(&writers, &edge_senders, &buf[..n]).await;
|
||||
total += n as u64;
|
||||
rate_limit_sleep(spec.max_bytes_per_sec, n).await;
|
||||
}
|
||||
|
||||
let d = if spec.tap {
|
||||
@@ -448,6 +453,7 @@ fn spawn_splitter(
|
||||
if n == 0 { break; }
|
||||
broadcast_chunk(&writers, &edge_senders, &buf[..n]).await;
|
||||
total += n as u64;
|
||||
rate_limit_sleep(spec.max_bytes_per_sec, n).await;
|
||||
}
|
||||
debug!(bytes = total, consumers = writers.len(), "splitter finished");
|
||||
|
||||
@@ -469,6 +475,19 @@ fn spawn_splitter(
|
||||
SplitterHandle { handle }
|
||||
}
|
||||
|
||||
/// Token-bucket simple: si `max_bps > 0`, sleep `chunk_size / max_bps`
|
||||
/// segundos. Implementación crude pero suficiente para v1.
|
||||
async fn rate_limit_sleep(max_bps: u64, chunk_bytes: usize) {
|
||||
if max_bps == 0 {
|
||||
return;
|
||||
}
|
||||
let secs = chunk_bytes as f64 / max_bps as f64;
|
||||
let ms = (secs * 1000.0) as u64;
|
||||
if ms > 0 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(ms)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn broadcast_chunk(
|
||||
writers: &[AsyncFd<std::os::fd::OwnedFd>],
|
||||
edge_senders: &[Option<crate::flow_channel::FlowSender>],
|
||||
@@ -721,6 +740,7 @@ mod tests {
|
||||
enrich_producer: true,
|
||||
replay_chunks: 32,
|
||||
replay_bytes: 0,
|
||||
max_bytes_per_sec: 0,
|
||||
},
|
||||
restart_on_failure: false,
|
||||
restart_backoff_ms: 200,
|
||||
|
||||
Reference in New Issue
Block a user