tawasuyu/brahman
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(shipote): data plane + DAG fan-in/out + stats + lifecycle (fases F-I)

Browse Source 
Pipeline runtime:
- Fan-out 1→N (splitter task replica al N consumers) y fan-in N→1 (merger
  task con mpsc + reader-per-input). DAGs no lineales soportados.
- Flow channels: Unix socket + tokio broadcast con replay buffer
  configurable por pipeline (DiscernPolicy.replay_chunks). Subscribers
  externos vía `shipote flow tail <socket>`.
- Templating en specs con `${KEY}` (CLI `--var KEY=VALUE`). Walk
  recursivo sobre serde_json::Value, soporta todos los strings del schema.
- Pipelines guardados (`pipeline save/saved-list/drop/run-saved`)
  persisten con el snapshot.

Lifecycle de comandos:
- Log capture per-stream (stdout/stderr separados) via pipe O_CLOEXEC +
  AsyncFd. CLI `shipote logs <ws> <cmd> --stream {stdout,stderr,both}`.
- Stop graceful con tiempo configurable: SIGTERM → grace → SIGKILL.
  Tanto a nivel workspace como pipeline individual.
- TTL auto-stop ya existente (Fase C) sigue funcionando.

ente-incarnate:
- ChildStdio declarativo (Fase C) + ChildPreExec declarativo nuevo:
  NoNewPrivs, ParentDeathSig, Dumpable, NewSession, Chdir, Umask.
- Aplicación pre-execve async-signal-safe en ambos paths (plain via
  Command::pre_exec, namespaced via callback del clone(2)).

Observabilidad:
- WorkspaceStats: RSS + RSS peak (VmHWM o memory.peak cgroup) + CPU usec
  + uptime. Fuente per-proc o cgroup según delegation.
- shipote-shell con sparkline ASCII por workspace (history cap 24),
  card de flow channels activos, vista de comandos + saved pipelines.
- Tap → broker: cada edge enriquecido con TypeRef se anuncia como Card
  efímera vía SidecarPool (graceful si broker no corre).

Discern:
- Integrado en yahweh-provider-fs (mime_type en EntityNode).
- Integrado en nouser-core::cluster::pick_lens como fallback cuando la
  extensión cae a Lens::Grid.

79 tests pasan: ente-incarnate (16), nouser-core (27), shipote-card (8),
shipote-core (20), shipote-discern (5), yahweh-provider-fs (3).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
sergio
2026-05-11 00:29:46 +00:00
parent c22d2480b9
commit 36dac00c8d
13 changed files with 2187 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/apps/shipote-shell/src/main.rs
+221 -2
View File
@@ -6,7 +6,8 @@
use gpui::{div, prelude::*, px, Context, IntoElement, Render, SharedString, Window};
use shipote_protocol::{
default_socket_path, read_frame, write_frame, CommandInfo, Request, Response, WorkspaceSummary,
default_socket_path, read_frame, write_frame, CommandInfo, FlowInfo, Request, Response,
WorkspaceStatsInfo, WorkspaceSummary,
};
use std::path::PathBuf;
use std::time::Duration;
@@ -42,10 +43,16 @@ struct Shell {
/// Comandos por workspace, indexados por workspace id.toString().
commands: std::collections::BTreeMap<String, Vec<CommandInfo>>,
saved_pipelines: Vec<String>,
flows: Vec<FlowInfo>,
/// History de RSS por workspace (últimas N samples).
stats_history: std::collections::BTreeMap<String, std::collections::VecDeque<WorkspaceStatsInfo>>,
caps: Option<CapsSummary>,
last_probe_ms: u64,
recent_log: Option<(String, String)>,
}
const STATS_HISTORY_LEN: usize = 24;
fn main() {
launch_app("Shipote — Shell", (820., 560.), Shell::new);
}
@@ -71,14 +78,36 @@ impl Shell {
me.workspaces = snap.workspaces;
me.commands = snap.commands;
me.saved_pipelines = snap.saved_pipelines;
me.flows = snap.flows;
// Append a la history por workspace.
for (ws_id, fresh) in &snap.fresh_stats {
let h = me
.stats_history
.entry(ws_id.clone())
.or_default();
if h.len() >= STATS_HISTORY_LEN {
h.pop_front();
}
h.push_back(fresh.clone());
}
// Limpiar history de workspaces que ya no existen.
let alive: std::collections::HashSet<String> = me
.workspaces
.iter()
.map(|w| w.id.to_string())
.collect();
me.stats_history.retain(|k, _| alive.contains(k));
me.caps = Some(snap.caps);
me.recent_log = snap.recent_log;
}
Err(reason) => {
me.state = DaemonState::Down { reason };
me.workspaces.clear();
me.commands.clear();
me.saved_pipelines.clear();
me.flows.clear();
me.caps = None;
me.recent_log = None;
}
}
me.last_probe_ms = elapsed;
@@ -95,8 +124,11 @@ impl Shell {
workspaces: Vec::new(),
commands: std::collections::BTreeMap::new(),
saved_pipelines: Vec::new(),
flows: Vec::new(),
stats_history: std::collections::BTreeMap::new(),
caps: None,
last_probe_ms: 0,
recent_log: None,
}
}
}
@@ -106,7 +138,12 @@ struct Snapshot {
workspaces: Vec<WorkspaceSummary>,
commands: std::collections::BTreeMap<String, Vec<CommandInfo>>,
saved_pipelines: Vec<String>,
flows: Vec<FlowInfo>,
/// Stats fresco por workspace (id.toString → stats).
fresh_stats: std::collections::BTreeMap<String, WorkspaceStatsInfo>,
caps: CapsSummary,
/// tail del log del comando más reciente (label + bytes). None si no hay.
recent_log: Option<(String, String)>,
}
fn probe_blocking(path: &std::path::Path) -> Result<Snapshot, String> {
@@ -132,6 +169,7 @@ fn probe_blocking(path: &std::path::Path) -> Result<Snapshot, String> {
// Commands por workspace.
let mut commands_map = std::collections::BTreeMap::new();
let mut fresh_stats = std::collections::BTreeMap::new();
for w in &workspaces {
write_frame(&mut stream, &Request::CommandList { workspace: w.id })
.await
@@ -144,6 +182,16 @@ fn probe_blocking(path: &std::path::Path) -> Result<Snapshot, String> {
commands_map.insert(w.id.to_string(), items);
}
}
// Stats por workspace.
write_frame(&mut stream, &Request::WorkspaceStats { workspace: w.id })
.await
.map_err(|e| format!("write stats: {e}"))?;
let resp: Response = read_frame(&mut stream)
.await
.map_err(|e| format!("read stats: {e}"))?;
if let Response::WorkspaceStats { info } = resp {
fresh_stats.insert(w.id.to_string(), info);
}
}
// Saved pipelines.
@@ -158,6 +206,68 @@ fn probe_blocking(path: &std::path::Path) -> Result<Snapshot, String> {
_ => Vec::new(),
};
// Flow channels activos (data plane).
write_frame(&mut stream, &Request::FlowList)
.await
.map_err(|e| format!("write flows: {e}"))?;
let resp: Response = read_frame(&mut stream)
.await
.map_err(|e| format!("read flows: {e}"))?;
let flows = match resp {
Response::FlowList { items } => items,
_ => Vec::new(),
};
// Live tail: log del comando más reciente con bytes>0.
let recent_log = {
// Pick: comando con id más alto que tiene log_bytes>0, en cualquier workspace.
let mut best: Option<(&str, &CommandInfo)> = None;
for (ws, cmds) in &commands_map {
for c in cmds {
if c.log_bytes == 0 {
continue;
}
let take = match &best {
Some((_, prev)) => c.id > prev.id,
None => true,
};
if take {
best = Some((ws.as_str(), c));
}
}
}
match best {
Some((ws_str, cmd)) => {
let ws_id: shipote_card::WorkspaceId = ws_str
.parse::<ulid::Ulid>()
.map(shipote_card::WorkspaceId)
.map_err(|e| format!("ulid parse: {e}"))?;
write_frame(
&mut stream,
&Request::CommandLogs {
workspace: ws_id,
command: cmd.id,
tail_bytes: 512,
stream: "both".into(),
},
)
.await
.map_err(|e| format!("write logs: {e}"))?;
let resp: Response = read_frame(&mut stream)
.await
.map_err(|e| format!("read logs: {e}"))?;
match resp {
Response::CommandLogs { bytes } => {
let text = String::from_utf8_lossy(&bytes).to_string();
Some((cmd.label.clone(), text))
}
_ => None,
}
}
None => None,
}
};
write_frame(&mut stream, &Request::Capabilities)
.await
.map_err(|e| format!("write caps: {e}"))?;
@@ -182,11 +292,36 @@ fn probe_blocking(path: &std::path::Path) -> Result<Snapshot, String> {
workspaces,
commands: commands_map,
saved_pipelines,
flows,
fresh_stats,
caps,
recent_log,
})
})
}
/// Render ASCII de sparkline para una serie de valores. `chars` define los
/// glifos (orden ascendente). Auto-scales al máximo de la serie.
fn sparkline(values: &[u64], width: usize) -> String {
if values.is_empty() {
return String::new();
}
const CHARS: &[char] = &['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
let take = values.len().min(width);
let series = &values[values.len() - take..];
let max = *series.iter().max().unwrap_or(&1);
if max == 0 {
return "▁".repeat(take);
}
series
.iter()
.map(|v| {
let idx = ((*v as f64 / max as f64) * (CHARS.len() as f64 - 1.0)).round() as usize;
CHARS[idx.min(CHARS.len() - 1)]
})
.collect()
}
impl Render for Shell {
fn render(&mut self, _w: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
let theme = Theme::global(cx).clone();
@@ -247,7 +382,28 @@ impl Render for Shell {
let ws_items: Vec<String> = self
.workspaces
.iter()
.map(|w| format!("{} {:<20} cmds={} uptime={}ms", w.id, w.label, w.commands, w.uptime_ms))
.map(|w| {
let key = w.id.to_string();
let history = self.stats_history.get(&key);
let rss_series: Vec<u64> = history
.map(|h| h.iter().map(|s| s.rss_bytes.unwrap_or(0)).collect())
.unwrap_or_default();
let spark = sparkline(&rss_series, STATS_HISTORY_LEN);
let (rss_now, peak) = history
.and_then(|h| h.back())
.map(|s| (s.rss_bytes.unwrap_or(0), s.rss_peak_bytes.unwrap_or(0)))
.unwrap_or((0, 0));
let rss_mb = rss_now as f64 / 1024.0 / 1024.0;
let peak_mb = peak as f64 / 1024.0 / 1024.0;
format!(
"{:<14} {:<14} {} {:>6.1}M peak {:>6.1}M",
&w.id.to_string()[20..],
w.label,
spark,
rss_mb,
peak_mb,
)
})
.collect();
let ws_count = self.workspaces.len().to_string();
let ws_descr = if self.workspaces.is_empty() {
@@ -294,6 +450,33 @@ impl Render for Shell {
"definiciones reusables vía run-saved".to_string()
};
// Flow channels (data plane).
let flow_count: usize = self.flows.iter().map(|f| f.sockets.len()).sum();
let flow_items: Vec<String> = self
.flows
.iter()
.flat_map(|f| {
let pipe = f.pipeline.to_string();
let short = &pipe[pipe.len() - 6..];
f.sockets
.iter()
.map(move |s| {
format!(
"{short} {}",
s.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| s.display().to_string())
)
})
.collect::<Vec<_>>()
})
.collect();
let flow_descr = if flow_count == 0 {
"pipelines con --tap exponen sockets aquí".to_string()
} else {
"shipote flow tail <socket> para suscribirse".to_string()
};
let body = div()
.flex()
.flex_col()
@@ -349,8 +532,44 @@ impl Render for Shell {
text,
text_dim,
&saved_items,
))
.child(stat_card(
cx,
"Flow channels",
flow_count.to_string(),
&flow_descr,
accent_up,
text,
text_dim,
&flow_items,
));
// Live tail del comando más reciente con output.
let live_card = self.recent_log.as_ref().map(|(label, content)| {
// Cortamos a las últimas ~12 líneas para no inflar el panel.
let lines: Vec<String> = content
.lines()
.rev()
.take(12)
.map(|l| l.to_string())
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
stat_card(
cx,
"Live tail",
label.clone(),
"últimas líneas del comando más reciente",
accent_up,
text,
text_dim,
&lines,
)
});
let body = body.when_some(live_card, |d, c| d.child(c));
div()
.flex()
.flex_col()