feat(shipote): data plane + DAG fan-in/out + stats + lifecycle (fases F-I)

Pipeline runtime:
- Fan-out 1→N (splitter task replica al N consumers) y fan-in N→1 (merger
  task con mpsc + reader-per-input). DAGs no lineales soportados.
- Flow channels: Unix socket + tokio broadcast con replay buffer
  configurable por pipeline (DiscernPolicy.replay_chunks). Subscribers
  externos vía `shipote flow tail <socket>`.
- Templating en specs con `${KEY}` (CLI `--var KEY=VALUE`). Walk
  recursivo sobre serde_json::Value, soporta todos los strings del schema.
- Pipelines guardados (`pipeline save/saved-list/drop/run-saved`)
  persisten con el snapshot.

Lifecycle de comandos:
- Log capture per-stream (stdout/stderr separados) via pipe O_CLOEXEC +
  AsyncFd. CLI `shipote logs <ws> <cmd> --stream {stdout,stderr,both}`.
- Stop graceful con tiempo configurable: SIGTERM → grace → SIGKILL.
  Tanto a nivel workspace como pipeline individual.
- TTL auto-stop ya existente (Fase C) sigue funcionando.

ente-incarnate:
- ChildStdio declarativo (Fase C) + ChildPreExec declarativo nuevo:
  NoNewPrivs, ParentDeathSig, Dumpable, NewSession, Chdir, Umask.
- Aplicación pre-execve async-signal-safe en ambos paths (plain via
  Command::pre_exec, namespaced via callback del clone(2)).

Observabilidad:
- WorkspaceStats: RSS + RSS peak (VmHWM o memory.peak cgroup) + CPU usec
  + uptime. Fuente per-proc o cgroup según delegation.
- shipote-shell con sparkline ASCII por workspace (history cap 24),
  card de flow channels activos, vista de comandos + saved pipelines.
- Tap → broker: cada edge enriquecido con TypeRef se anuncia como Card
  efímera vía SidecarPool (graceful si broker no corre).

Discern:
- Integrado en yahweh-provider-fs (mime_type en EntityNode).
- Integrado en nouser-core::cluster::pick_lens como fallback cuando la
  extensión cae a Lens::Grid.

79 tests pasan: ente-incarnate (16), nouser-core (27), shipote-card (8),
shipote-core (20), shipote-discern (5), yahweh-provider-fs (3).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
sergio
2026-05-11 00:29:46 +00:00
parent c22d2480b9
commit 36dac00c8d
13 changed files with 2187 additions and 253 deletions
+156 -8
View File
@@ -63,11 +63,31 @@ enum Cmd {
/// Bytes desde el final (0 = todo).
#[arg(long, default_value_t = 0)]
tail: usize,
/// Stream a leer: stdout | stderr | both.
#[arg(long, default_value = "both")]
stream: String,
},
/// Pipeline DAG con flujo tipado.
#[command(subcommand)]
Pipeline(PipeCmd),
/// Flow data plane (subscribirse a streams enriquecidos).
#[command(subcommand)]
Flow(FlowCmd),
}
#[derive(Subcommand, Debug)]
enum FlowCmd {
/// Listar pipelines activos con sus sockets de flow.
List,
/// Cerrar el data plane de un pipeline (drop de todos sus sockets).
Drop { pipeline: String },
/// Suscribirse a un flow socket y volcar bytes a stdout.
Tail {
/// Path al Unix socket del flow.
socket: PathBuf,
},
}
#[derive(Subcommand, Debug)]
@@ -80,6 +100,9 @@ enum PipeCmd {
/// discernir el TypeRef del flujo.
#[arg(long)]
tap: bool,
/// Variables `KEY=VALUE` para sustitución `${KEY}` en el spec.
#[arg(long = "var", value_parser = parse_kv)]
vars: Vec<(String, String)>,
},
/// Guardar un pipeline bajo un nombre (persiste con el snapshot).
Save {
@@ -90,16 +113,31 @@ enum PipeCmd {
},
/// Listar nombres de pipelines guardados.
SavedList,
/// Eliminar un pipeline guardado.
/// Eliminar un pipeline guardado (no afecta runs en curso).
Drop { name: String },
/// Detener un pipeline en curso por ID (SIGTERM → grace → SIGKILL
/// sólo a sus comandos).
Stop {
/// ULID del pipeline (devuelto por `pipeline run`).
pipeline: String,
#[arg(long, default_value_t = 1000)]
grace_ms: u64,
},
/// Ejecutar un pipeline guardado por nombre.
RunSaved {
name: String,
#[arg(long)]
tap: bool,
#[arg(long = "var", value_parser = parse_kv)]
vars: Vec<(String, String)>,
},
}
fn parse_kv(s: &str) -> Result<(String, String), String> {
let (k, v) = s.split_once('=').ok_or_else(|| format!("expected KEY=VALUE, got `{s}`"))?;
Ok((k.to_string(), v.to_string()))
}
#[derive(Subcommand, Debug)]
enum WsCmd {
/// Crear un workspace desde un spec TOML/JSON.
@@ -112,6 +150,13 @@ enum WsCmd {
/// Detener un workspace por ID.
Stop {
id: String,
/// Milisegundos de gracia tras SIGTERM antes de SIGKILL.
#[arg(long, default_value_t = 1000)]
grace_ms: u64,
},
/// Resource accounting (RSS, CPU, comandos vivos).
Stats {
id: String,
},
}
@@ -185,9 +230,33 @@ async fn main() -> Result<()> {
}
}
Cmd::Workspace(WsCmd::Stop { id }) => {
Cmd::Workspace(WsCmd::Stats { id }) => {
let id = parse_ws_id(&id)?;
let resp = round_trip(&mut stream, Request::WorkspaceStop { id }).await?;
let resp = round_trip(&mut stream, Request::WorkspaceStats { workspace: id }).await?;
match resp {
Response::WorkspaceStats { info } => {
println!("commands: {} alive / {} total", info.commands_alive, info.commands_total);
let fmt_mib = |b: u64| format!("{:.2} MiB", b as f64 / 1024.0 / 1024.0);
let rss = info.rss_bytes.map(fmt_mib).unwrap_or_else(|| "".into());
let peak = info.rss_peak_bytes.map(fmt_mib).unwrap_or_else(|| "".into());
let cpu = info
.cpu_usec
.map(|u| format!("{:.3} s", u as f64 / 1_000_000.0))
.unwrap_or_else(|| "".into());
println!("rss: {rss}");
println!("rss_peak: {peak}");
println!("cpu: {cpu}");
println!("source: {}", info.source);
println!("uptime: {} ms", info.uptime_ms);
}
Response::Error { message } => return Err(anyhow!(message)),
other => print_unexpected(&other),
}
}
Cmd::Workspace(WsCmd::Stop { id, grace_ms }) => {
let id = parse_ws_id(&id)?;
let resp = round_trip(&mut stream, Request::WorkspaceStop { id, grace_ms }).await?;
match resp {
Response::WorkspaceStopped { id, reaped } => {
println!("stopped {id} (reaped {reaped})");
@@ -218,9 +287,17 @@ async fn main() -> Result<()> {
}
}
Cmd::Pipeline(PipeCmd::Run { spec, tap }) => {
Cmd::Pipeline(PipeCmd::Run { spec, tap, vars }) => {
let p = load_pipeline_spec(&spec).with_context(|| format!("load {}", spec.display()))?;
let resp = round_trip(&mut stream, Request::PipelineRun { spec: p, tap }).await?;
let resp = round_trip(
&mut stream,
Request::PipelineRun {
spec: p,
tap,
vars: vars.into_iter().collect(),
},
)
.await?;
print_pipeline_started(resp)?;
}
@@ -249,6 +326,17 @@ async fn main() -> Result<()> {
}
}
Cmd::Pipeline(PipeCmd::Stop { pipeline, grace_ms }) => {
let pid = Ulid::from_string(&pipeline).map_err(|e| anyhow!("invalid pipeline id: {e}"))?;
let resp = round_trip(&mut stream, Request::PipelineStop { pipeline: pid, grace_ms }).await?;
match resp {
Response::PipelineStopped { pipeline, reaped } => {
println!("stopped pipeline {pipeline} (reaped {reaped})");
}
other => print_unexpected(&other),
}
}
Cmd::Pipeline(PipeCmd::Drop { name }) => {
let resp = round_trip(&mut stream, Request::PipelineDrop { name }).await?;
match resp {
@@ -263,8 +351,16 @@ async fn main() -> Result<()> {
}
}
Cmd::Pipeline(PipeCmd::RunSaved { name, tap }) => {
let resp = round_trip(&mut stream, Request::PipelineRunSaved { name, tap }).await?;
Cmd::Pipeline(PipeCmd::RunSaved { name, tap, vars }) => {
let resp = round_trip(
&mut stream,
Request::PipelineRunSaved {
name,
tap,
vars: vars.into_iter().collect(),
},
)
.await?;
print_pipeline_started(resp)?;
}
@@ -292,7 +388,7 @@ async fn main() -> Result<()> {
}
}
Cmd::Logs { workspace, command, tail } => {
Cmd::Logs { workspace, command, tail, stream: which_stream } => {
let ws = parse_ws_id(&workspace)?;
let cmd_id = Ulid::from_string(&command).map_err(|e| anyhow!("invalid command id: {e}"))?;
let resp = round_trip(
@@ -301,6 +397,7 @@ async fn main() -> Result<()> {
workspace: ws,
command: cmd_id,
tail_bytes: tail,
stream: which_stream,
},
)
.await?;
@@ -316,6 +413,57 @@ async fn main() -> Result<()> {
}
}
Cmd::Flow(FlowCmd::List) => {
let resp = round_trip(&mut stream, Request::FlowList).await?;
match resp {
Response::FlowList { items } => {
if items.is_empty() {
println!("(no active flows)");
}
for it in items {
println!("{}", it.pipeline);
for s in it.sockets {
println!(" {}", s.display());
}
}
}
other => print_unexpected(&other),
}
}
Cmd::Flow(FlowCmd::Drop { pipeline }) => {
let pid = Ulid::from_string(&pipeline).map_err(|e| anyhow!("invalid pipeline id: {e}"))?;
let resp = round_trip(&mut stream, Request::FlowDrop { pipeline: pid }).await?;
match resp {
Response::FlowDropped { pipeline, existed } => {
if existed {
println!("dropped {pipeline}");
} else {
eprintln!("no existía: {pipeline}");
}
}
other => print_unexpected(&other),
}
}
Cmd::Flow(FlowCmd::Tail { socket }) => {
// Subscribirse directo al socket — no pasamos por el daemon.
use tokio::io::AsyncReadExt;
let mut s = UnixStream::connect(&socket)
.await
.with_context(|| format!("connect {}", socket.display()))?;
let mut buf = [0u8; 4096];
loop {
let n = s.read(&mut buf).await?;
if n == 0 {
break;
}
use std::io::Write;
let _ = std::io::stdout().write_all(&buf[..n]);
let _ = std::io::stdout().flush();
}
}
Cmd::Discern { path } => {
let bytes = std::fs::read(&path).with_context(|| format!("read {}", path.display()))?;
// Sample: hasta 4 KiB.