Skip to main content

handshake_core/application/
traces.rs

1use super::{Application, ApplicationError, Inner};
2use crate::event::{EventKind, StateEvent};
3use crate::store::{NewTraceSpan, TraceEnding};
4use crate::terminal::TerminalObservation;
5use crate::{
6    TerminalId, TerminalStatus, TraceAnchor, TraceEventId, TraceEventKind, TraceEventsPage,
7    TraceSpanId, TraceSpanStatus, Workflow, WorkflowId, WorkflowTracePage,
8};
9
10#[derive(Clone, Debug)]
11pub(super) struct PendingTraceEnding {
12    observation: TerminalObservation,
13    status: TraceSpanStatus,
14    kind: TraceEventKind,
15    message: String,
16}
17
18impl PendingTraceEnding {
19    fn as_store_ending(&self, terminal_id: TerminalId) -> TraceEnding<'_> {
20        TraceEnding {
21            observed_at: self.observation.observed_at,
22            status: self.status,
23            kind: self.kind,
24            message: &self.message,
25            anchor: Some(TraceAnchor {
26                terminal_id,
27                byte_offset: self.observation.byte_offset,
28                boundary_sizes: self.observation.boundary_sizes.clone(),
29            }),
30        }
31    }
32}
33
34impl Application {
35    /// Reads cached trace storage statistics and schedules background maintenance.
36    ///
37    /// # Errors
38    /// Returns storage or application-lock failures.
39    pub fn trace_storage_status(
40        &self,
41        span: Option<TraceSpanId>,
42    ) -> Result<crate::TraceStorageStatus, ApplicationError> {
43        let inner = self.lock_inner()?;
44        let status = inner.folders.read_store().storage_status(span)?;
45        let request = inner.folders.read_store().maintenance_request(false);
46        drop(inner);
47        if let Some(request) = request {
48            self.history
49                .lock()
50                .map_err(|_| ApplicationError::Poisoned)?
51                .request(request);
52        }
53        Ok(status)
54    }
55
56    /// Protects a step's full retained details from automatic cleanup.
57    ///
58    /// # Errors
59    /// Returns storage or application-lock failures.
60    pub fn pin_trace_details(
61        &self,
62        span: TraceSpanId,
63        pinned: bool,
64    ) -> Result<(), ApplicationError> {
65        self.lock_inner()?
66            .folders
67            .store()
68            .pin_trace_details(span, pinned)?;
69        Ok(())
70    }
71
72    /// Clears unpinned, completed full details in the background. Native files are untouched.
73    ///
74    /// # Errors
75    /// Returns storage or application-lock failures.
76    pub fn clear_trace_details(&self) -> Result<(), ApplicationError> {
77        self.lock_inner()?.folders.store().request_trace_clear()?;
78        if let Some(request) = self
79            .lock_inner()?
80            .folders
81            .read_store()
82            .maintenance_request(true)
83        {
84            self.history
85                .lock()
86                .map_err(|_| ApplicationError::Poisoned)?
87                .request(request);
88        }
89        Ok(())
90    }
91    /// Reads full recorded activity input or output in bounded UTF-8 pages.
92    ///
93    /// # Errors
94    /// Returns an error for missing activity, invalid page size, or unavailable payload storage.
95    pub fn trace_detail(
96        &self,
97        activity: crate::TraceActivityId,
98        output: bool,
99        offset: u64,
100        limit: usize,
101    ) -> Result<crate::TraceDetailPage, ApplicationError> {
102        Ok(self
103            .lock_inner()?
104            .folders
105            .read_store()
106            .trace_detail(activity, output, offset, limit)?)
107    }
108    /// Reads native tool and subagent activity in stable ID order. Refresh from the beginning when
109    /// the workflow revision changes, because a finish updates an existing activity.
110    ///
111    /// # Errors
112    /// Returns an error if the span is missing, the page limit is invalid, or storage fails.
113    pub fn trace_activities(
114        &self,
115        span_id: TraceSpanId,
116        after: Option<crate::TraceActivityId>,
117        limit: usize,
118    ) -> Result<crate::TraceActivitiesPage, ApplicationError> {
119        self.poll_harness_steps()?;
120        if let Some(request) = self
121            .lock_inner()?
122            .folders
123            .read_store()
124            .history_request(span_id)?
125        {
126            self.history
127                .lock()
128                .map_err(|_| ApplicationError::Poisoned)?
129                .request(request);
130        }
131        Ok(self
132            .lock_inner()?
133            .folders
134            .read_store()
135            .trace_activities(span_id, after, limit)?)
136    }
137
138    pub(super) fn poll_shell_observations(&self) -> Result<(), ApplicationError> {
139        let mut inner = self.lock_inner()?;
140        inner.record_shell_observations(Vec::new());
141        if inner.pending_shell_marks.is_empty() {
142            inner.record_shell_observations(self.terminal_output.take_shell_observations()?);
143        }
144        inner.finish_shell_process_endings(&self.terminal_output);
145        Ok(())
146    }
147
148    /// Reads a bounded page of the workflow's durable trace, including closed workflow history.
149    ///
150    /// # Errors
151    /// Returns an error if the workflow is missing, the page limit is invalid, or storage fails.
152    pub fn workflow_trace(
153        &self,
154        workflow_id: WorkflowId,
155        before: Option<TraceSpanId>,
156        limit: usize,
157    ) -> Result<WorkflowTracePage, ApplicationError> {
158        self.poll_harness_steps()?;
159        self.poll_shell_observations()?;
160        let inner = self.lock_inner()?;
161        let mut page = inner
162            .folders
163            .read_store()
164            .workflow_trace(workflow_id, before, limit)?;
165        for span in &mut page.spans {
166            let live_assignment = page
167                .lanes
168                .iter()
169                .find(|lane| lane.lane_id == span.lane_id)
170                .and_then(|lane| lane.agent_id)
171                .is_some_and(|agent_id| {
172                    inner.workflows.workflows.iter().any(|workflow| {
173                        workflow.workflow_id == workflow_id
174                            && workflow.run.as_ref().is_some_and(|run| {
175                                run.status == crate::RunStatus::Running
176                                    && run
177                                        .active_agents()
178                                        .iter()
179                                        .any(|agent| agent.agent_id == agent_id.0)
180                            })
181                            && workflow.agents.iter().any(|agent| {
182                                agent.agent_id == agent_id
183                                    && span.terminal_id.is_none_or(|id| agent.terminal_id == id)
184                            })
185                    })
186                });
187            span.is_live = span.status == TraceSpanStatus::Running
188                && (live_assignment
189                    || span.terminal_id.is_some_and(|id| {
190                        inner.trace_spans.get(&id) == Some(&span.span_id)
191                            && inner.terminals.get(&id) == Some(&TerminalStatus::Running)
192                    }));
193        }
194        Ok(page)
195    }
196
197    /// Reads a bounded page of events for a span, in durable event ID order.
198    ///
199    /// # Errors
200    /// Returns an error if the span is missing, the page limit is invalid, or storage fails.
201    pub fn trace_events(
202        &self,
203        span_id: TraceSpanId,
204        after: Option<TraceEventId>,
205        limit: usize,
206    ) -> Result<TraceEventsPage, ApplicationError> {
207        self.poll_harness_steps()?;
208        Ok(self
209            .lock_inner()?
210            .folders
211            .read_store()
212            .trace_events(span_id, after, limit)?)
213    }
214}
215
216impl Inner {
217    pub(super) fn record_shell_observations(
218        &mut self,
219        observations: Vec<crate::terminal::ShellObservation>,
220    ) {
221        for item in &observations {
222            if let Some(workflow) = self.workflows.workflows.iter().find(|workflow| {
223                workflow.kind == crate::WorkflowKind::Terminal
224                    && workflow.terminal_id == item.terminal_id
225            }) {
226                self.pending_shell_workflows
227                    .entry(item.terminal_id)
228                    .or_insert_with(|| workflow.clone());
229            }
230        }
231        self.pending_shell_marks.extend(observations);
232        while let Some(item) = self.pending_shell_marks.front().cloned() {
233            if let Err(error) = self.record_shell_mark(item) {
234                tracing::warn!(%error, "couldn't persist shell command boundary");
235                break;
236            }
237            self.pending_shell_marks.pop_front();
238        }
239        self.pending_shell_workflows.retain(|id, _| {
240            self.pending_shell_marks
241                .iter()
242                .any(|item| item.terminal_id == *id)
243        });
244    }
245
246    pub(super) fn defer_shell_process_ending(
247        &mut self,
248        id: TerminalId,
249        observation: &TerminalObservation,
250        status: TraceSpanStatus,
251        kind: TraceEventKind,
252        message: &str,
253    ) -> bool {
254        if !observation.integrated_shell
255            && !self.command_shells.contains(&id)
256            && !self
257                .pending_shell_marks
258                .iter()
259                .any(|item| item.terminal_id == id)
260        {
261            return false;
262        }
263        self.shell_process_endings.insert(
264            id,
265            PendingTraceEnding {
266                observation: observation.clone(),
267                status,
268                kind,
269                message: message.into(),
270            },
271        );
272        true
273    }
274
275    pub(super) fn finish_shell_process_endings(
276        &mut self,
277        output: &crate::terminal::TerminalStream,
278    ) {
279        let finished = self
280            .shell_process_endings
281            .iter()
282            .filter(|(id, _)| {
283                output.reader_finished(**id)
284                    && !self
285                        .pending_shell_marks
286                        .iter()
287                        .any(|item| item.terminal_id == **id)
288            })
289            .map(|(&id, ending)| (id, ending.clone()))
290            .collect::<Vec<_>>();
291        for (id, mut ending) in finished {
292            if self.command_shells.contains(&id) {
293                ending.status = TraceSpanStatus::Stopped;
294                ending.kind = TraceEventKind::WorkflowEvent;
295                ending.message = "Shell ended before the command's ending was recorded.".into();
296            }
297            if let Err(error) = self.end_trace(
298                id,
299                ending.observation,
300                ending.status,
301                ending.kind,
302                &ending.message,
303            ) {
304                tracing::warn!(%error, "couldn't persist shell process ending");
305            } else {
306                self.shell_process_endings.remove(&id);
307                self.command_shells.remove(&id);
308            }
309        }
310    }
311
312    fn record_shell_mark(
313        &mut self,
314        item: crate::terminal::ShellObservation,
315    ) -> Result<(), ApplicationError> {
316        use crate::terminal::ShellMark;
317        let terminal_id = item.terminal_id;
318        let Some(workflow) = self
319            .workflows
320            .workflows
321            .iter()
322            .find(|workflow| {
323                workflow.kind == crate::WorkflowKind::Terminal
324                    && workflow.terminal_id == terminal_id
325            })
326            .cloned()
327            .or_else(|| self.pending_shell_workflows.get(&terminal_id).cloned())
328        else {
329            return Ok(());
330        };
331        if !matches!(&item.mark, ShellMark::Stopped(_))
332            && !self.command_shells.contains(&terminal_id)
333        {
334            if let Some(span) = self.trace_spans.get(&terminal_id).copied() {
335                self.folders.store().activate_command_trace(span)?;
336                self.trace_spans.remove(&terminal_id);
337            }
338            self.command_shells.insert(terminal_id);
339        }
340        match item.mark {
341            ShellMark::Prompt => {}
342            ShellMark::CommandStart(title) => {
343                self.end_trace(
344                    terminal_id,
345                    item.observation.clone(),
346                    TraceSpanStatus::Stopped,
347                    TraceEventKind::WorkflowEvent,
348                    "Command ending was not recorded.",
349                )?;
350                let mut span = new_span(&workflow);
351                span.title = &title;
352                span.started_at = item.observation.observed_at;
353                span.anchor = Some(TraceAnchor {
354                    terminal_id,
355                    byte_offset: item.observation.byte_offset,
356                    boundary_sizes: item.observation.boundary_sizes,
357                });
358                let id = self.folders.store().start_command_trace(&span)?;
359                self.trace_spans.insert(terminal_id, id);
360            }
361            ShellMark::CommandEnd(code) => {
362                self.end_trace(
363                    terminal_id,
364                    item.observation,
365                    if code == 0 {
366                        TraceSpanStatus::Completed
367                    } else {
368                        TraceSpanStatus::Failed
369                    },
370                    TraceEventKind::WorkflowEvent,
371                    &format!("Command exited with code {code}."),
372                )?;
373            }
374            ShellMark::Lost => {
375                self.end_trace(
376                    terminal_id,
377                    item.observation.clone(),
378                    TraceSpanStatus::Stopped,
379                    TraceEventKind::WorkflowEvent,
380                    "Shell integration boundaries were lost.",
381                )?;
382                let mut span = new_span(&workflow);
383                span.started_at = item.observation.observed_at;
384                span.anchor = Some(TraceAnchor {
385                    terminal_id,
386                    byte_offset: item.observation.byte_offset,
387                    boundary_sizes: item.observation.boundary_sizes,
388                });
389                let id = self.folders.store().start_trace_span(&span)?;
390                self.trace_spans.insert(terminal_id, id);
391                self.command_shells.remove(&terminal_id);
392            }
393            ShellMark::Stopped(message) => {
394                self.end_trace(
395                    terminal_id,
396                    item.observation,
397                    TraceSpanStatus::Stopped,
398                    TraceEventKind::WorkflowEvent,
399                    &message,
400                )?;
401                self.command_shells.remove(&terminal_id);
402            }
403        }
404        // A journal failure must not retry an already committed command boundary.
405        if let Err(error) = self.publish_trace(workflow.workflow_id) {
406            tracing::warn!(%error, "couldn't publish shell trace revision");
407        }
408        Ok(())
409    }
410
411    pub(super) fn start_trace(&mut self, workflow: &Workflow) -> Result<(), ApplicationError> {
412        self.folders.store().remember_terminals(workflow)?;
413        if workflow.kind == crate::WorkflowKind::Draft {
414            return Ok(());
415        }
416        if workflow.kind == crate::WorkflowKind::Agents {
417            let keys = workflow
418                .agents
419                .iter()
420                .map(|agent| format!("agent:{}", agent.agent_id.0))
421                .collect::<Vec<_>>();
422            let spans = workflow
423                .agents
424                .iter()
425                .zip(&keys)
426                .filter(|(agent, _)| agent.terminal_id.value() != 0)
427                .map(|(agent, key)| NewTraceSpan {
428                    workflow_id: workflow.workflow_id,
429                    lane_key: key,
430                    lane_name: &agent.role,
431                    is_agent: true,
432                    role: Some(&agent.role),
433                    harness: None,
434                    title: "Shell",
435                    started_at: workflow.started_at,
436                    anchor: Some(TraceAnchor {
437                        terminal_id: agent.terminal_id,
438                        byte_offset: 0,
439                        boundary_sizes: Some(Vec::new()),
440                    }),
441                })
442                .collect::<Vec<_>>();
443            let ids = self.folders.store().start_trace_spans(&spans)?;
444            for (span, id) in spans.iter().zip(ids) {
445                if let Some(anchor) = &span.anchor {
446                    self.trace_spans.insert(anchor.terminal_id, id);
447                }
448            }
449            return self.publish_trace(workflow.workflow_id);
450        }
451        let span_id = self.folders.store().start_trace_span(&new_span(workflow))?;
452        self.trace_spans.insert(workflow.terminal_id, span_id);
453        self.publish_trace(workflow.workflow_id)
454    }
455
456    pub(super) fn start_agent_trace(
457        &mut self,
458        workflow: &Workflow,
459        harness: crate::HarnessId,
460        placeholder: TerminalId,
461        observation: Option<TerminalObservation>,
462        session: Option<&str>,
463    ) -> Result<(), ApplicationError> {
464        let ending = self
465            .pending_trace_endings
466            .get(&placeholder)
467            .cloned()
468            .or_else(|| {
469                observation.map(|observation| PendingTraceEnding {
470                    observation,
471                    status: TraceSpanStatus::Stopped,
472                    kind: TraceEventKind::ProcessStopped,
473                    message: "Shell stopped when its agent started.".to_owned(),
474                })
475            });
476        let ending = ending
477            .as_ref()
478            .map(|ending| ending.as_store_ending(placeholder));
479        let placeholder_span = self.trace_spans.get(&placeholder).copied();
480        let span_id = self.folders.store().start_agent_trace(
481            &new_span(workflow),
482            harness,
483            placeholder_span.zip(ending.as_ref()),
484            session,
485        )?;
486        self.trace_spans.remove(&placeholder);
487        self.pending_trace_endings.remove(&placeholder);
488        self.trace_spans.insert(workflow.terminal_id, span_id);
489        self.publish_trace(workflow.workflow_id)
490    }
491
492    pub(super) fn stop_trace(
493        &mut self,
494        terminal_id: TerminalId,
495        observation: TerminalObservation,
496        message: &str,
497    ) -> Result<(), ApplicationError> {
498        self.stop_pending_shell_trace(terminal_id, &observation, message);
499        self.end_trace(
500            terminal_id,
501            observation,
502            TraceSpanStatus::Stopped,
503            TraceEventKind::ProcessStopped,
504            message,
505        )
506    }
507
508    fn stop_pending_shell_trace(
509        &mut self,
510        terminal_id: TerminalId,
511        observation: &TerminalObservation,
512        message: &str,
513    ) {
514        self.shell_process_endings.remove(&terminal_id);
515        if self
516            .pending_shell_marks
517            .iter()
518            .any(|item| item.terminal_id == terminal_id)
519        {
520            self.pending_shell_marks
521                .push_back(crate::terminal::ShellObservation {
522                    terminal_id,
523                    mark: crate::terminal::ShellMark::Stopped(message.into()),
524                    observation: observation.clone(),
525                });
526        } else {
527            self.command_shells.remove(&terminal_id);
528        }
529    }
530
531    pub(super) fn end_trace(
532        &mut self,
533        terminal_id: TerminalId,
534        observation: TerminalObservation,
535        status: TraceSpanStatus,
536        kind: TraceEventKind,
537        message: &str,
538    ) -> Result<(), ApplicationError> {
539        let Some(span_id) = self.trace_spans.get(&terminal_id).copied() else {
540            if let Some(&workflow_id) = self.step_terminals.get(&terminal_id) {
541                let ending = self
542                    .pending_trace_endings
543                    .entry(terminal_id)
544                    .or_insert_with(|| PendingTraceEnding {
545                        observation,
546                        status,
547                        kind,
548                        message: message.into(),
549                    });
550                self.folders.store().record_harness_process_ending(
551                    workflow_id,
552                    terminal_id,
553                    &ending.as_store_ending(terminal_id),
554                )?;
555                self.pending_trace_endings.remove(&terminal_id);
556                self.publish_trace(workflow_id)?;
557            }
558            return Ok(());
559        };
560        let ending = self
561            .pending_trace_endings
562            .entry(terminal_id)
563            .or_insert_with(|| PendingTraceEnding {
564                observation,
565                status,
566                kind,
567                message: message.to_owned(),
568            });
569        let workflow_id = self
570            .folders
571            .store()
572            .finish_trace_span(span_id, &ending.as_store_ending(terminal_id))?;
573        self.trace_spans.remove(&terminal_id);
574        self.pending_trace_endings.remove(&terminal_id);
575        if let Some(id) = workflow_id {
576            self.publish_trace(id)?;
577        }
578        Ok(())
579    }
580
581    pub(super) fn close_workflow_trace(
582        &mut self,
583        workflow_id: WorkflowId,
584        observations: &[(TerminalId, TerminalObservation)],
585    ) -> Result<(), ApplicationError> {
586        let idle_endings: Vec<_> = observations
587            .iter()
588            .filter(|(terminal_id, _)| {
589                self.step_terminals.contains_key(terminal_id)
590                    && !self.trace_spans.contains_key(terminal_id)
591            })
592            .map(|(terminal_id, observation)| {
593                (
594                    *terminal_id,
595                    TraceEnding {
596                        observed_at: observation.observed_at,
597                        status: TraceSpanStatus::Stopped,
598                        kind: TraceEventKind::ProcessStopped,
599                        message: "Process stopped when its workflow was closed.",
600                        anchor: Some(TraceAnchor {
601                            terminal_id: *terminal_id,
602                            byte_offset: observation.byte_offset,
603                            boundary_sizes: observation.boundary_sizes.clone(),
604                        }),
605                    },
606                )
607            })
608            .collect();
609        let pending = observations
610            .iter()
611            .filter_map(|(terminal_id, observation)| {
612                self.trace_spans.get(terminal_id).map(|span_id| {
613                    let ending = self
614                        .pending_trace_endings
615                        .get(terminal_id)
616                        .cloned()
617                        .unwrap_or_else(|| PendingTraceEnding {
618                            observation: observation.clone(),
619                            status: TraceSpanStatus::Stopped,
620                            kind: TraceEventKind::ProcessStopped,
621                            message: "Process stopped when its workflow was closed.".to_owned(),
622                        });
623                    (*terminal_id, *span_id, ending)
624                })
625            })
626            .collect::<Vec<_>>();
627        let endings = pending
628            .iter()
629            .map(|(terminal_id, span_id, ending)| (*span_id, ending.as_store_ending(*terminal_id)))
630            .collect::<Vec<_>>();
631        let changed = self.folders.store().close_workflow_and_trace(
632            workflow_id,
633            crate::workflow::timestamp(),
634            &endings,
635            &idle_endings,
636        )?;
637        for (terminal_id, _) in observations {
638            self.trace_spans.remove(terminal_id);
639            self.pending_trace_endings.remove(terminal_id);
640        }
641        for (terminal_id, observation) in observations {
642            self.stop_pending_shell_trace(
643                *terminal_id,
644                observation,
645                "Command stopped when its workflow was closed.",
646            );
647        }
648        if changed {
649            self.publish_trace(workflow_id)?;
650        }
651        Ok(())
652    }
653
654    pub(super) fn retry_trace_endings(&mut self) {
655        let pending = self.pending_trace_endings.clone();
656        for (id, ending) in pending {
657            if let Err(error) = self.end_trace(
658                id,
659                ending.observation,
660                ending.status,
661                ending.kind,
662                &ending.message,
663            ) {
664                tracing::warn!(%error, "failed to retry observed trace ending");
665            }
666        }
667    }
668
669    pub(super) fn publish_trace(&mut self, id: WorkflowId) -> Result<(), ApplicationError> {
670        let summary = self.folders.store().trace_summary(id)?;
671        self.events
672            .append(EventKind::State(StateEvent::TraceChanged(summary)))?;
673        Ok(())
674    }
675}
676
677fn new_span(workflow: &Workflow) -> NewTraceSpan<'_> {
678    let is_agent = workflow.kind == crate::WorkflowKind::SingleAgent;
679    NewTraceSpan {
680        workflow_id: workflow.workflow_id,
681        lane_key: if is_agent { "agent" } else { "terminal" },
682        lane_name: if is_agent { "Agent" } else { "Terminal" },
683        is_agent,
684        role: is_agent.then_some("agent"),
685        harness: workflow.harness.map(|harness| harness.definition().name),
686        title: if is_agent { &workflow.name } else { "Shell" },
687        started_at: workflow.started_at,
688        anchor: Some(TraceAnchor {
689            terminal_id: workflow.terminal_id,
690            byte_offset: 0,
691            boundary_sizes: Some(Vec::new()),
692        }),
693    }
694}
695
696#[cfg(test)]
697mod tests {
698    use std::thread;
699    use std::time::{Duration, Instant};
700
701    use super::*;
702    use crate::{
703        Command, CommandDisposition, RequestId, TerminalSize, WorkflowKind, WorkflowStatus,
704    };
705
706    fn create(application: &Application, folder: &std::path::Path) -> Workflow {
707        application
708            .handle_command(
709                RequestId(1),
710                Command::OpenFolder {
711                    path: folder.to_owned(),
712                },
713            )
714            .unwrap();
715        let receipt = application
716            .handle_command(
717                RequestId(2),
718                Command::CreateWorkflow {
719                    folder: folder.to_owned(),
720                    session_id: None,
721                    kind: WorkflowKind::Terminal,
722                    roles: Vec::new(),
723                    size: TerminalSize {
724                        rows: 24,
725                        columns: 80,
726                        pixel_width: 800,
727                        pixel_height: 480,
728                    },
729                },
730            )
731            .unwrap();
732        assert_eq!(receipt.disposition, CommandDisposition::Accepted);
733        application.snapshot().unwrap().workflows.workflows[0].clone()
734    }
735
736    fn reject_trace_writes(application: &Application) {
737        application.lock_inner().unwrap().folders.store().execute_test_sql(
738            "CREATE TRIGGER reject_trace BEFORE INSERT ON trace_events BEGIN SELECT RAISE(FAIL, 'test recording failure'); END;"
739        );
740    }
741
742    fn allow_trace_writes(application: &Application) {
743        application
744            .lock_inner()
745            .unwrap()
746            .folders
747            .store()
748            .execute_test_sql("DROP TRIGGER reject_trace");
749    }
750
751    // Local macOS/Codex runs: Bash tests timing out here with one live `Shell` span
752    // can inherit Apple Terminal's session-saving EXIT trap from
753    // /etc/bashrc_Apple_Terminal (TERM_PROGRAM=Apple_Terminal and TERM_SESSION_ID).
754    // Our shell/bash hook preserves existing traps by using the lifetime-span fallback.
755    // Clear the inherited session ID for the test command; temporary HOME alone cannot
756    // prevent the system startup script from installing that trap. From the repo root:
757    //   env -u TERM_SESSION_ID make test-lib LIB_TEST_ARGS='application::traces::tests::bash_'
758    //   env -u TERM_SESSION_ID make check-lib
759    // This scopes the workaround to the test process and its children. Tests still need
760    // permission to spawn shells, open PTYs, and write their temporary profiles.
761    fn wait_shell_trace(
762        application: &Application,
763        workflow: &Workflow,
764        ready: impl Fn(&WorkflowTracePage) -> bool,
765    ) -> WorkflowTracePage {
766        let deadline = Instant::now() + Duration::from_secs(10);
767        loop {
768            while let Some(chunk) = application.next_terminal_chunk().unwrap() {
769                // Fish probes device attributes before its first prompt. Stand in for the
770                // terminal emulator so PTY integration tests exercise actual interactive hooks.
771                if chunk.bytes.windows(4).any(|bytes| bytes == b"\x1b[0c") {
772                    application
773                        .write_terminal_input(workflow.terminal_id, b"\x1b[?62;4c")
774                        .unwrap();
775                }
776            }
777            let page = application
778                .workflow_trace(workflow.workflow_id, None, 200)
779                .unwrap();
780            if ready(&page) {
781                return page;
782            }
783            assert!(Instant::now() < deadline, "shell trace timed out: {page:?}");
784            thread::sleep(Duration::from_millis(5));
785        }
786    }
787
788    fn check_command_shell(shell: &str) {
789        let folder = tempfile::tempdir().unwrap();
790        let home = tempfile::tempdir().unwrap();
791        let profile = "printf 'HANDSHAKE_STARTUP\\n'\n";
792        for file in [".zshenv", ".zprofile", ".zshrc", ".zlogin", ".bash_profile"] {
793            std::fs::write(home.path().join(file), profile).unwrap();
794        }
795        std::fs::create_dir_all(home.path().join(".config/fish")).unwrap();
796        std::fs::write(home.path().join(".config/fish/config.fish"), profile).unwrap();
797        let mut application = Application::with_event_capacity(4096).unwrap();
798        application.terminals.set_test_shell(shell.into());
799        application
800            .terminals
801            .set_test_shell_home(home.path().to_owned());
802        let workflow = create(&application, folder.path());
803        wait_shell_trace(&application, &workflow, |page| page.spans.is_empty());
804        let commands = [
805            (
806                "printf 'HANDSHAKE_OUTPUT ☃\\n'",
807                TraceSpanStatus::Completed,
808                0,
809            ),
810            ("false", TraceSpanStatus::Failed, 1),
811            (
812                "printf 'pipe\\n' | cat; true",
813                TraceSpanStatus::Completed,
814                0,
815            ),
816            ("false", TraceSpanStatus::Failed, 1),
817        ];
818        for (index, (command, status, code)) in commands.iter().enumerate() {
819            application
820                .write_terminal_input(workflow.terminal_id, format!("{command}\n").as_bytes())
821                .unwrap();
822            let page = wait_shell_trace(&application, &workflow, |page| {
823                page.spans.len() == index + 1 && page.spans[0].ended_at.is_some()
824            });
825            let span = &page.spans[0];
826            assert_eq!(&span.title, command, "{shell}");
827            assert_eq!(&span.status, status, "{shell}");
828            assert!(!span.is_live);
829            let events = application
830                .trace_events(span.span_id, None, 10)
831                .unwrap()
832                .events;
833            assert_eq!(events.len(), 2, "{shell}: {events:?}");
834            assert_eq!(events[0].message, "Command started.");
835            assert_eq!(
836                events[1].message,
837                format!("Command exited with code {code}.")
838            );
839            let start = events[0].anchor.as_ref().unwrap();
840            let end = events[1].anchor.as_ref().unwrap();
841            assert!(end.byte_offset > start.byte_offset);
842            assert_eq!(start.terminal_id, workflow.terminal_id);
843            let crate::TranscriptRead::Output(page) = application
844                .read_terminal_transcript(workflow.terminal_id, 0, crate::MAX_TRANSCRIPT_READ_BYTES)
845                .unwrap()
846            else {
847                panic!("missing transcript");
848            };
849            assert!(page.replay_available);
850            assert!(String::from_utf8_lossy(&page.bytes).contains(";handshake=handshake-shell-"));
851            assert!(page.bytes[..usize::try_from(start.byte_offset).unwrap()].ends_with(b"\x07"));
852            if index == 0 {
853                assert!(
854                    String::from_utf8_lossy(
855                        &page.bytes[usize::try_from(start.byte_offset).unwrap()
856                            ..usize::try_from(end.byte_offset).unwrap()]
857                    )
858                    .contains("HANDSHAKE_OUTPUT ☃")
859                );
860                application
861                    .write_terminal_input(
862                        workflow.terminal_id,
863                        if shell == "/bin/bash" {
864                            b"\n# comment only\n"
865                        } else {
866                            b"\n"
867                        },
868                    )
869                    .unwrap();
870            }
871        }
872        application
873            .write_terminal_input(workflow.terminal_id, b"exit 7\n")
874            .unwrap();
875        let page = wait_shell_trace(&application, &workflow, |page| {
876            page.spans.len() == 5 && page.spans[0].ended_at.is_some()
877        });
878        assert_eq!(page.spans[0].title, "exit 7");
879        assert_eq!(page.spans[0].status, TraceSpanStatus::Failed);
880        for file in [".zshenv", ".zprofile", ".zshrc", ".zlogin", ".bash_profile"] {
881            assert_eq!(
882                std::fs::read_to_string(home.path().join(file)).unwrap(),
883                profile
884            );
885        }
886    }
887
888    #[test]
889    fn zsh_commands_have_titles_statuses_and_output_anchors() {
890        check_command_shell("/bin/zsh");
891    }
892
893    #[test]
894    fn bash_commands_have_titles_statuses_and_output_anchors() {
895        check_command_shell("/bin/bash");
896    }
897
898    #[test]
899    fn fish_commands_have_titles_statuses_and_output_anchors() {
900        let shell = [
901            "/opt/homebrew/bin/fish",
902            "/usr/local/bin/fish",
903            "/usr/bin/fish",
904        ]
905        .into_iter()
906        .find(|path| std::path::Path::new(path).is_file());
907        if let Some(shell) = shell {
908            check_command_shell(shell);
909        } else {
910            assert!(
911                std::env::var_os("HANDSHAKE_REQUIRE_FISH_TESTS").is_none(),
912                "install fish to run the shell integration tests"
913            );
914        }
915    }
916
917    #[test]
918    fn bash_disabled_history_uses_one_lifetime_span() {
919        let folder = tempfile::tempdir().unwrap();
920        let home = tempfile::tempdir().unwrap();
921        std::fs::write(home.path().join(".bash_profile"), "HISTSIZE=0\n").unwrap();
922        let mut application = Application::with_event_capacity(4096).unwrap();
923        application.terminals.set_test_shell("/bin/bash".into());
924        application
925            .terminals
926            .set_test_shell_home(home.path().to_owned());
927        let workflow = create(&application, folder.path());
928        application
929            .write_terminal_input(workflow.terminal_id, b"false\nexit 7\n")
930            .unwrap();
931        let page = wait_shell_trace(&application, &workflow, |page| {
932            page.spans[0].ended_at.is_some()
933        });
934        assert_eq!(page.spans.len(), 1);
935        assert_eq!(page.spans[0].title, "Shell");
936        assert_eq!(page.spans[0].status, TraceSpanStatus::Exited);
937    }
938
939    #[test]
940    fn bash_history_disabled_during_a_session_falls_back_once() {
941        let folder = tempfile::tempdir().unwrap();
942        let home = tempfile::tempdir().unwrap();
943        let mut application = Application::with_event_capacity(4096).unwrap();
944        application.terminals.set_test_shell("/bin/bash".into());
945        application
946            .terminals
947            .set_test_shell_home(home.path().to_owned());
948        let workflow = create(&application, folder.path());
949        wait_shell_trace(&application, &workflow, |page| page.spans.is_empty());
950        application
951            .write_terminal_input(workflow.terminal_id, b"HISTSIZE=0\n")
952            .unwrap();
953        wait_shell_trace(&application, &workflow, |page| {
954            page.spans.len() == 1 && page.spans[0].ended_at.is_some()
955        });
956        application
957            .write_terminal_input(workflow.terminal_id, b"true\nfalse\nexit\n")
958            .unwrap();
959        let page = wait_shell_trace(&application, &workflow, |page| {
960            page.spans.len() == 2 && page.spans[0].ended_at.is_some()
961        });
962        assert_eq!(page.spans[0].title, "Shell");
963        assert_eq!(page.spans[1].title, "HISTSIZE=0");
964        assert_eq!(page.spans[1].status, TraceSpanStatus::Completed);
965    }
966
967    fn command_observation(
968        workflow: &Workflow,
969        mark: crate::terminal::ShellMark,
970    ) -> crate::terminal::ShellObservation {
971        crate::terminal::ShellObservation {
972            terminal_id: workflow.terminal_id,
973            mark,
974            observation: TerminalObservation {
975                integrated_shell: true,
976                observed_at: workflow.started_at + 1,
977                byte_offset: 100,
978                boundary_sizes: Some(Vec::new()),
979            },
980        }
981    }
982
983    #[test]
984    fn delayed_command_start_is_persisted_and_stopped_after_workflow_close() {
985        check_delayed_command_close(false);
986    }
987
988    #[test]
989    fn delayed_command_start_is_persisted_and_stopped_after_folder_close() {
990        check_delayed_command_close(true);
991    }
992
993    fn check_delayed_command_close(close_folder: bool) {
994        use crate::terminal::ShellMark;
995        let folder = tempfile::tempdir().unwrap();
996        let mut application = Application::with_event_capacity(4096).unwrap();
997        application.terminals.set_test_shell("/bin/sh".into());
998        let workflow = create(&application, folder.path());
999        application
1000            .lock_inner()
1001            .unwrap()
1002            .record_shell_observations(vec![command_observation(&workflow, ShellMark::Prompt)]);
1003        reject_trace_writes(&application);
1004        application
1005            .lock_inner()
1006            .unwrap()
1007            .record_shell_observations(vec![command_observation(
1008                &workflow,
1009                ShellMark::CommandStart("delayed".into()),
1010            )]);
1011        application
1012            .handle_command(
1013                RequestId(3),
1014                if close_folder {
1015                    Command::CloseFolder
1016                } else {
1017                    Command::CloseWorkflow {
1018                        workflow_id: workflow.workflow_id,
1019                    }
1020                },
1021            )
1022            .unwrap();
1023        assert_eq!(application.snapshot().unwrap().workflows.workflows, []);
1024        allow_trace_writes(&application);
1025        let page = application
1026            .workflow_trace(workflow.workflow_id, None, 10)
1027            .unwrap();
1028        assert_eq!(page.spans.len(), 1);
1029        assert_eq!(page.spans[0].title, "delayed");
1030        assert_eq!(page.spans[0].status, TraceSpanStatus::Stopped);
1031        assert_eq!(
1032            application
1033                .trace_events(page.spans[0].span_id, None, 10)
1034                .unwrap()
1035                .events
1036                .len(),
1037            2
1038        );
1039        assert!(
1040            application
1041                .lock_inner()
1042                .unwrap()
1043                .pending_shell_workflows
1044                .is_empty()
1045        );
1046    }
1047
1048    #[test]
1049    fn failed_command_ending_survives_folder_close_and_storage_recovery() {
1050        use crate::terminal::ShellMark;
1051        let folder = tempfile::tempdir().unwrap();
1052        let mut application = Application::with_event_capacity(4096).unwrap();
1053        application.terminals.set_test_shell("/bin/sh".into());
1054        let workflow = create(&application, folder.path());
1055        application
1056            .lock_inner()
1057            .unwrap()
1058            .record_shell_observations(vec![
1059                command_observation(&workflow, ShellMark::Prompt),
1060                command_observation(
1061                    &workflow,
1062                    ShellMark::CommandStart("completed before close".into()),
1063                ),
1064            ]);
1065        reject_trace_writes(&application);
1066        application
1067            .lock_inner()
1068            .unwrap()
1069            .record_shell_observations(vec![command_observation(
1070                &workflow,
1071                ShellMark::CommandEnd(0),
1072            )]);
1073        application
1074            .handle_command(RequestId(3), Command::CloseFolder)
1075            .unwrap();
1076        allow_trace_writes(&application);
1077        let page = application
1078            .workflow_trace(workflow.workflow_id, None, 10)
1079            .unwrap();
1080        assert_eq!(page.spans.len(), 1);
1081        assert_eq!(page.spans[0].title, "completed before close");
1082        assert_eq!(page.spans[0].status, TraceSpanStatus::Completed);
1083        assert_eq!(
1084            application
1085                .trace_events(page.spans[0].span_id, None, 10)
1086                .unwrap()
1087                .events
1088                .len(),
1089            2
1090        );
1091    }
1092
1093    #[test]
1094    fn failed_integration_loss_preserves_its_fallback_when_the_folder_closes() {
1095        use crate::terminal::ShellMark;
1096        let folder = tempfile::tempdir().unwrap();
1097        let mut application = Application::with_event_capacity(4096).unwrap();
1098        application.terminals.set_test_shell("/bin/sh".into());
1099        let workflow = create(&application, folder.path());
1100        application
1101            .lock_inner()
1102            .unwrap()
1103            .record_shell_observations(vec![command_observation(
1104                &workflow,
1105                ShellMark::CommandStart("running".into()),
1106            )]);
1107        reject_trace_writes(&application);
1108        application
1109            .lock_inner()
1110            .unwrap()
1111            .record_shell_observations(vec![command_observation(&workflow, ShellMark::Lost)]);
1112        application
1113            .handle_command(RequestId(3), Command::CloseFolder)
1114            .unwrap();
1115        allow_trace_writes(&application);
1116        let page = application
1117            .workflow_trace(workflow.workflow_id, None, 10)
1118            .unwrap();
1119        assert_eq!(page.spans.len(), 2);
1120        assert_eq!(page.spans[0].title, "Shell");
1121        assert_eq!(page.spans[1].title, "running");
1122        assert!(
1123            page.spans
1124                .iter()
1125                .all(|span| span.status == TraceSpanStatus::Stopped)
1126        );
1127    }
1128
1129    #[test]
1130    fn integrated_process_exit_defers_before_its_first_mark_and_preserves_fallback_status() {
1131        use crate::terminal::{ShellMark, TerminalStream, TranscriptRecorder};
1132        let folder = tempfile::tempdir().unwrap();
1133        let mut application = Application::with_event_capacity(4096).unwrap();
1134        application.terminals.set_test_shell("/bin/sh".into());
1135        let workflow = create(&application, folder.path());
1136        let observation = command_observation(&workflow, ShellMark::Prompt).observation;
1137        let finished = TerminalStream::new(
1138            4096,
1139            4,
1140            std::sync::Arc::new(TranscriptRecorder::temporary().unwrap()),
1141        )
1142        .unwrap();
1143        let mut inner = application.lock_inner().unwrap();
1144        assert!(inner.defer_shell_process_ending(
1145            workflow.terminal_id,
1146            &observation,
1147            TraceSpanStatus::Failed,
1148            TraceEventKind::ProcessFailed,
1149            "exit 7"
1150        ));
1151        inner.finish_shell_process_endings(&application.terminal_output);
1152        assert!(
1153            inner
1154                .shell_process_endings
1155                .contains_key(&workflow.terminal_id)
1156        );
1157        inner.record_shell_observations(vec![
1158            command_observation(&workflow, ShellMark::Prompt),
1159            command_observation(&workflow, ShellMark::CommandStart("exit 7".into())),
1160            command_observation(&workflow, ShellMark::CommandEnd(7)),
1161        ]);
1162        inner.finish_shell_process_endings(&finished);
1163        drop(inner);
1164        let page = application
1165            .workflow_trace(workflow.workflow_id, None, 10)
1166            .unwrap();
1167        assert_eq!(page.spans[0].title, "exit 7");
1168        assert_eq!(page.spans[0].status, TraceSpanStatus::Failed);
1169
1170        let second_folder = tempfile::tempdir().unwrap();
1171        let mut fallback = Application::with_event_capacity(4096).unwrap();
1172        fallback.terminals.set_test_shell("/bin/sh".into());
1173        let second = create(&fallback, second_folder.path());
1174        let mut inner = fallback.lock_inner().unwrap();
1175        inner.defer_shell_process_ending(
1176            second.terminal_id,
1177            &observation,
1178            TraceSpanStatus::Exited,
1179            TraceEventKind::ProcessExited,
1180            "exit 0",
1181        );
1182        inner.finish_shell_process_endings(&finished);
1183        drop(inner);
1184        let page = fallback
1185            .workflow_trace(second.workflow_id, None, 10)
1186            .unwrap();
1187        assert_eq!(page.spans[0].title, "Shell");
1188        assert_eq!(page.spans[0].status, TraceSpanStatus::Exited);
1189    }
1190
1191    #[test]
1192    fn command_boundaries_retry_in_order_after_a_store_failure() {
1193        use crate::terminal::{ShellMark, ShellObservation};
1194        let folder = tempfile::tempdir().unwrap();
1195        let application = Application::with_event_capacity(4096).unwrap();
1196        let workflow = create(&application, folder.path());
1197        reject_trace_writes(&application);
1198        let marks = [
1199            ShellMark::Prompt,
1200            ShellMark::CommandStart("first command".into()),
1201            ShellMark::CommandEnd(1),
1202            ShellMark::CommandStart("second command".into()),
1203            ShellMark::CommandEnd(0),
1204        ]
1205        .into_iter()
1206        .enumerate()
1207        .map(|(i, mark)| ShellObservation {
1208            terminal_id: workflow.terminal_id,
1209            mark,
1210            observation: TerminalObservation {
1211                integrated_shell: false,
1212                observed_at: workflow.started_at + i as u64,
1213                byte_offset: i as u64 + 100,
1214                boundary_sizes: Some(Vec::new()),
1215            },
1216        })
1217        .collect();
1218        application
1219            .lock_inner()
1220            .unwrap()
1221            .record_shell_observations(marks);
1222        assert_eq!(
1223            application.lock_inner().unwrap().pending_shell_marks.len(),
1224            4
1225        );
1226        allow_trace_writes(&application);
1227        let page = application
1228            .workflow_trace(workflow.workflow_id, None, 10)
1229            .unwrap();
1230        assert_eq!(page.spans.len(), 2);
1231        assert_eq!(page.spans[0].title, "second command");
1232        assert_eq!(page.spans[0].status, TraceSpanStatus::Completed);
1233        assert_eq!(page.spans[1].title, "first command");
1234        assert_eq!(page.spans[1].status, TraceSpanStatus::Failed);
1235        for span in page.spans {
1236            let events = application
1237                .trace_events(span.span_id, None, 10)
1238                .unwrap()
1239                .events;
1240            assert_eq!(events.len(), 2);
1241            assert_eq!(events[0].timestamp, span.started_at);
1242            assert_eq!(events[1].timestamp, span.ended_at.unwrap());
1243        }
1244    }
1245
1246    #[test]
1247    fn shell_marks_do_not_create_command_spans_in_agent_terminals() {
1248        let folder = tempfile::tempdir().unwrap();
1249        let application = Application::with_event_capacity(4096).unwrap();
1250        let workflow = create_agents(&application, folder.path()).unwrap();
1251        application
1252            .lock_inner()
1253            .unwrap()
1254            .record_shell_observations(vec![crate::terminal::ShellObservation {
1255                terminal_id: workflow.agents[0].terminal_id,
1256                mark: crate::terminal::ShellMark::CommandStart("not a terminal workflow".into()),
1257                observation: TerminalObservation {
1258                    integrated_shell: false,
1259                    observed_at: workflow.started_at,
1260                    byte_offset: 42,
1261                    boundary_sizes: Some(Vec::new()),
1262                },
1263            }]);
1264        let page = application
1265            .workflow_trace(workflow.workflow_id, None, 10)
1266            .unwrap();
1267        assert_eq!(page.spans.len(), 2);
1268        assert!(page.spans.iter().all(|span| span.title == "Shell"));
1269    }
1270
1271    fn create_agents(
1272        application: &Application,
1273        folder: &std::path::Path,
1274    ) -> Result<Workflow, ApplicationError> {
1275        application.handle_command(
1276            RequestId(1),
1277            Command::OpenFolder {
1278                path: folder.to_owned(),
1279            },
1280        )?;
1281        application.handle_command(
1282            RequestId(2),
1283            Command::CreateWorkflow {
1284                folder: folder.to_owned(),
1285                session_id: None,
1286                kind: WorkflowKind::Agents,
1287                roles: vec!["Worker".to_owned(), "Worker".to_owned()],
1288                size: TerminalSize {
1289                    rows: 24,
1290                    columns: 80,
1291                    pixel_width: 800,
1292                    pixel_height: 480,
1293                },
1294            },
1295        )?;
1296        Ok(application.snapshot()?.workflows.workflows[0].clone())
1297    }
1298
1299    #[test]
1300    fn agents_with_the_same_role_have_distinct_lanes_and_exit_independently() {
1301        let folder = tempfile::tempdir().unwrap();
1302        let application = Application::with_event_capacity(4096).unwrap();
1303        let workflow = create_agents(&application, folder.path()).unwrap();
1304        let page = application
1305            .workflow_trace(workflow.workflow_id, None, 10)
1306            .unwrap();
1307        assert_eq!(page.summary.agent_count, 2);
1308        assert_eq!(page.lanes.len(), 2);
1309        assert!(
1310            page.lanes
1311                .iter()
1312                .all(|lane| lane.is_agent && lane.role.as_deref() == Some("Worker"))
1313        );
1314        assert_ne!(page.spans[0].lane_id, page.spans[1].lane_id);
1315        assert!(
1316            page.spans
1317                .iter()
1318                .all(|span| span.is_live && span.started_at == workflow.started_at)
1319        );
1320        let first = workflow.agents[0].terminal_id;
1321        let second = workflow.agents[1].terminal_id;
1322        application.lock_inner().unwrap().record_terminal_exit_at(
1323            first,
1324            Ok(crate::TerminalExit {
1325                exit_code: 3,
1326                signal: None,
1327            }),
1328            TerminalObservation {
1329                integrated_shell: false,
1330                observed_at: workflow.started_at + 10,
1331                byte_offset: 42,
1332                boundary_sizes: Some(Vec::new()),
1333            },
1334        );
1335        let workflows = application.snapshot().unwrap().workflows.workflows;
1336        assert_eq!(workflows[0].status, WorkflowStatus::Running);
1337        let page = application
1338            .workflow_trace(workflow.workflow_id, None, 10)
1339            .unwrap();
1340        assert_eq!(page.spans.iter().filter(|span| span.is_live).count(), 1);
1341        let span = page
1342            .spans
1343            .iter()
1344            .find(|span| span.terminal_id == Some(first))
1345            .unwrap();
1346        assert_eq!(span.status, TraceSpanStatus::Exited);
1347        let events = application
1348            .trace_events(span.span_id, None, 10)
1349            .unwrap()
1350            .events;
1351        assert_eq!(events[1].anchor.as_ref().unwrap().byte_offset, 42);
1352        application.lock_inner().unwrap().record_terminal_exit_at(
1353            second,
1354            Ok(crate::TerminalExit {
1355                exit_code: 0,
1356                signal: None,
1357            }),
1358            TerminalObservation {
1359                integrated_shell: false,
1360                observed_at: workflow.started_at + 20,
1361                byte_offset: 70,
1362                boundary_sizes: Some(Vec::new()),
1363            },
1364        );
1365        let snapshot = application.snapshot().unwrap();
1366        assert_eq!(
1367            snapshot.workflows.workflows[0].status,
1368            WorkflowStatus::Exited
1369        );
1370        assert_eq!(
1371            snapshot.workflows.workflows[0].ended_at,
1372            Some(workflow.started_at + 20)
1373        );
1374        application
1375            .handle_command(
1376                RequestId(3),
1377                Command::CloseWorkflow {
1378                    workflow_id: workflow.workflow_id,
1379                },
1380            )
1381            .unwrap();
1382        let page = application
1383            .workflow_trace(workflow.workflow_id, None, 10)
1384            .unwrap();
1385        assert_eq!(page.spans.len(), 2);
1386        assert!(
1387            page.spans
1388                .iter()
1389                .all(|span| !span.is_live && span.status == TraceSpanStatus::Exited)
1390        );
1391        for span in page.spans {
1392            assert_eq!(
1393                application
1394                    .trace_events(span.span_id, None, 10)
1395                    .unwrap()
1396                    .events
1397                    .len(),
1398                2
1399            );
1400        }
1401    }
1402
1403    #[test]
1404    fn failed_multi_agent_close_rolls_back_every_ending_and_then_closes_all_processes() {
1405        let folder = tempfile::tempdir().unwrap();
1406        let application = Application::with_event_capacity(4096).unwrap();
1407        let workflow = create_agents(&application, folder.path()).unwrap();
1408        application.lock_inner().unwrap().folders.store().execute_test_sql(
1409            "CREATE TRIGGER reject_trace BEFORE INSERT ON trace_events WHEN NEW.kind = 'processStopped' AND NEW.span_id = 2 BEGIN SELECT RAISE(FAIL, 'test second ending failure'); END;"
1410        );
1411        assert!(
1412            application
1413                .handle_command(
1414                    RequestId(3),
1415                    Command::CloseWorkflow {
1416                        workflow_id: workflow.workflow_id
1417                    }
1418                )
1419                .is_err()
1420        );
1421        let page = application
1422            .workflow_trace(workflow.workflow_id, None, 10)
1423            .unwrap();
1424        assert!(
1425            page.spans
1426                .iter()
1427                .all(|span| span.is_live && span.status == TraceSpanStatus::Running)
1428        );
1429        for span in &page.spans {
1430            assert_eq!(
1431                application
1432                    .trace_events(span.span_id, None, 10)
1433                    .unwrap()
1434                    .events
1435                    .len(),
1436                1
1437            );
1438        }
1439        assert_eq!(
1440            application
1441                .lock_inner()
1442                .unwrap()
1443                .folders
1444                .read_store()
1445                .workflows(folder.path())
1446                .unwrap()
1447                .len(),
1448            1
1449        );
1450        allow_trace_writes(&application);
1451        application
1452            .handle_command(
1453                RequestId(4),
1454                Command::CloseWorkflow {
1455                    workflow_id: workflow.workflow_id,
1456                },
1457            )
1458            .unwrap();
1459        let page = application
1460            .workflow_trace(workflow.workflow_id, None, 10)
1461            .unwrap();
1462        assert!(
1463            page.spans
1464                .iter()
1465                .all(|span| !span.is_live && span.status == TraceSpanStatus::Stopped)
1466        );
1467        for span in &page.spans {
1468            assert_eq!(
1469                application
1470                    .trace_events(span.span_id, None, 10)
1471                    .unwrap()
1472                    .events
1473                    .len(),
1474                2
1475            );
1476        }
1477        assert_eq!(application.snapshot().unwrap().terminals, []);
1478        for agent in workflow.agents {
1479            assert!(
1480                application
1481                    .write_terminal_input(agent.terminal_id, b"echo alive\n")
1482                    .is_err()
1483            );
1484        }
1485    }
1486
1487    #[test]
1488    fn failed_multi_agent_start_rolls_back_all_spans_and_stops_every_process() {
1489        let folder = tempfile::tempdir().unwrap();
1490        let application = Application::with_event_capacity(4096).unwrap();
1491        application.lock_inner().unwrap().folders.store().execute_test_sql(
1492            "CREATE TRIGGER reject_trace BEFORE INSERT ON trace_events WHEN NEW.kind = 'processStarted' AND NEW.span_id = 2 BEGIN SELECT RAISE(FAIL, 'test second start failure'); END;"
1493        );
1494        assert!(create_agents(&application, folder.path()).is_err());
1495        let snapshot = application.snapshot().unwrap();
1496        assert_eq!(snapshot.terminals, []);
1497        assert_eq!(snapshot.workflows.workflows, []);
1498        let page = application.workflow_trace(WorkflowId(1), None, 10).unwrap();
1499        assert_eq!(page.spans, []);
1500        assert_eq!(page.lanes, []);
1501        for id in [1, 2] {
1502            assert!(
1503                application
1504                    .terminals
1505                    .size(TerminalId::from_value(id))
1506                    .is_none()
1507            );
1508        }
1509    }
1510
1511    #[test]
1512    fn restoring_agents_reuses_lane_identity_and_cleans_up_all_failed_trace_starts() {
1513        let folder = tempfile::tempdir().unwrap();
1514        let application = Application::with_event_capacity(4096).unwrap();
1515        let workflow = create_agents(&application, folder.path()).unwrap();
1516        let initial = application
1517            .workflow_trace(workflow.workflow_id, None, 10)
1518            .unwrap();
1519        application
1520            .handle_command(RequestId(3), Command::CloseFolder)
1521            .unwrap();
1522        application.lock_inner().unwrap().folders.store().execute_test_sql(
1523            "CREATE TRIGGER reject_trace BEFORE INSERT ON trace_events WHEN NEW.kind = 'processStarted' AND NEW.span_id = 4 BEGIN SELECT RAISE(FAIL, 'test second restored start failure'); END;"
1524        );
1525        application
1526            .handle_command(
1527                RequestId(4),
1528                Command::OpenFolder {
1529                    path: folder.path().to_owned(),
1530                },
1531            )
1532            .unwrap();
1533        let snapshot = application.snapshot().unwrap();
1534        assert_eq!(snapshot.terminals, []);
1535        let failed = &snapshot.workflows.workflows[0];
1536        assert_eq!(failed.status, WorkflowStatus::Failed);
1537        assert!(
1538            failed
1539                .agents
1540                .iter()
1541                .all(|agent| agent.terminal_id.value() == 0)
1542        );
1543        let page = application
1544            .workflow_trace(workflow.workflow_id, None, 10)
1545            .unwrap();
1546        assert_eq!(page.spans.len(), 2);
1547        assert_eq!(page.lanes, initial.lanes);
1548        allow_trace_writes(&application);
1549        application
1550            .handle_command(RequestId(5), Command::CloseFolder)
1551            .unwrap();
1552        application
1553            .handle_command(
1554                RequestId(6),
1555                Command::OpenFolder {
1556                    path: folder.path().to_owned(),
1557                },
1558            )
1559            .unwrap();
1560        let page = application
1561            .workflow_trace(workflow.workflow_id, None, 10)
1562            .unwrap();
1563        assert_eq!(page.lanes, initial.lanes);
1564        assert_eq!(page.spans.len(), 4);
1565        assert_eq!(page.spans.iter().filter(|span| span.is_live).count(), 2);
1566    }
1567
1568    #[test]
1569    fn trace_write_failure_never_prevents_folder_process_cleanup() {
1570        let folder = tempfile::tempdir().unwrap();
1571        let application = Application::with_event_capacity(4096).unwrap();
1572        let workflow = create(&application, folder.path());
1573        reject_trace_writes(&application);
1574        application
1575            .handle_command(RequestId(3), Command::CloseFolder)
1576            .unwrap();
1577        let snapshot = application.snapshot().unwrap();
1578        assert!(snapshot.folders.open_folder.is_none());
1579        assert_eq!(snapshot.workflows.workflows, []);
1580        assert_eq!(snapshot.terminals, []);
1581        assert!(
1582            application
1583                .write_terminal_input(workflow.terminal_id, b"echo alive\n")
1584                .is_err()
1585        );
1586        allow_trace_writes(&application);
1587        application
1588            .handle_command(RequestId(4), Command::Ping)
1589            .unwrap();
1590        let page = application
1591            .workflow_trace(workflow.workflow_id, None, 10)
1592            .unwrap();
1593        assert_eq!(page.spans[0].status, TraceSpanStatus::Stopped);
1594        assert_eq!(
1595            application
1596                .trace_events(page.spans[0].span_id, None, 10)
1597                .unwrap()
1598                .events
1599                .len(),
1600            2
1601        );
1602    }
1603
1604    #[test]
1605    fn shutdown_retries_endings_even_after_folder_cleanup_removes_the_process() {
1606        let data = tempfile::tempdir().unwrap();
1607        let folder = tempfile::tempdir().unwrap();
1608        let mut application =
1609            Application::with_config(data.path(), crate::config::Config::default()).unwrap();
1610        application.terminals.set_test_shell("/bin/sh".into());
1611        let workflow = create(&application, folder.path());
1612        reject_trace_writes(&application);
1613        application
1614            .handle_command(RequestId(3), Command::CloseFolder)
1615            .unwrap();
1616        assert_eq!(
1617            application
1618                .lock_inner()
1619                .unwrap()
1620                .pending_trace_endings
1621                .len(),
1622            1
1623        );
1624        allow_trace_writes(&application);
1625        drop(application);
1626        let application =
1627            Application::with_config(data.path(), crate::config::Config::default()).unwrap();
1628        let page = application
1629            .workflow_trace(workflow.workflow_id, None, 10)
1630            .unwrap();
1631        assert_eq!(page.spans[0].status, TraceSpanStatus::Stopped);
1632        let events = application
1633            .trace_events(page.spans[0].span_id, None, 10)
1634            .unwrap()
1635            .events;
1636        assert_eq!(events.len(), 2);
1637        assert!(events[1].message.contains("folder was closed"));
1638    }
1639
1640    #[test]
1641    fn failed_restoration_stops_the_process_and_publishes_a_visible_failed_tab() {
1642        let folder = tempfile::tempdir().unwrap();
1643        let application = Application::with_event_capacity(4096).unwrap();
1644        let workflow = create(&application, folder.path());
1645        application
1646            .handle_command(RequestId(3), Command::CloseFolder)
1647            .unwrap();
1648        reject_trace_writes(&application);
1649        application
1650            .handle_command(
1651                RequestId(4),
1652                Command::OpenFolder {
1653                    path: folder.path().to_owned(),
1654                },
1655            )
1656            .unwrap();
1657        let snapshot = application.snapshot().unwrap();
1658        assert_eq!(snapshot.workflows.workflows.len(), 1);
1659        assert_eq!(
1660            snapshot.workflows.workflows[0].status,
1661            WorkflowStatus::Failed
1662        );
1663        assert_eq!(snapshot.workflows.workflows[0].terminal_id.value(), 0);
1664        assert_eq!(snapshot.terminals, []);
1665        allow_trace_writes(&application);
1666        application
1667            .handle_command(RequestId(5), Command::CloseFolder)
1668            .unwrap();
1669        application
1670            .handle_command(
1671                RequestId(6),
1672                Command::OpenFolder {
1673                    path: folder.path().to_owned(),
1674                },
1675            )
1676            .unwrap();
1677        let snapshot = application.snapshot().unwrap();
1678        assert_eq!(snapshot.terminals.len(), 1);
1679        assert_eq!(
1680            snapshot.workflows.workflows[0].workflow_id,
1681            workflow.workflow_id
1682        );
1683        assert_eq!(
1684            snapshot.workflows.workflows[0].status,
1685            WorkflowStatus::Running
1686        );
1687        assert_eq!(snapshot.traces[0].span_count, 2);
1688    }
1689
1690    #[test]
1691    fn a_failed_exit_write_is_retried_with_its_original_observation() {
1692        let folder = tempfile::tempdir().unwrap();
1693        let application = Application::with_event_capacity(4096).unwrap();
1694        let workflow = create(&application, folder.path());
1695        reject_trace_writes(&application);
1696        let at = workflow.started_at + 25;
1697        application.lock_inner().unwrap().record_terminal_exit_at(
1698            workflow.terminal_id,
1699            Ok(crate::TerminalExit {
1700                exit_code: 7,
1701                signal: None,
1702            }),
1703            TerminalObservation {
1704                integrated_shell: false,
1705                observed_at: at,
1706                byte_offset: 42,
1707                boundary_sizes: Some(Vec::new()),
1708            },
1709        );
1710        assert_eq!(
1711            application
1712                .lock_inner()
1713                .unwrap()
1714                .pending_trace_endings
1715                .len(),
1716            1
1717        );
1718        allow_trace_writes(&application);
1719        application
1720            .handle_command(
1721                RequestId(3),
1722                Command::CloseWorkflow {
1723                    workflow_id: workflow.workflow_id,
1724                },
1725            )
1726            .unwrap();
1727        let page = application
1728            .workflow_trace(workflow.workflow_id, None, 10)
1729            .unwrap();
1730        let events = application
1731            .trace_events(page.spans[0].span_id, None, 10)
1732            .unwrap()
1733            .events;
1734        assert_eq!(events.len(), 2);
1735        assert_eq!(events[1].kind, TraceEventKind::ProcessExited);
1736        assert!(events[1].message.contains("code 7"));
1737        assert_eq!(events[1].timestamp, at);
1738        assert_eq!(events[1].anchor.as_ref().unwrap().byte_offset, 42);
1739    }
1740
1741    #[test]
1742    fn failed_workflow_close_rolls_back_its_trace_ending() {
1743        let folder = tempfile::tempdir().unwrap();
1744        let application = Application::with_event_capacity(4096).unwrap();
1745        let workflow = create(&application, folder.path());
1746        application.lock_inner().unwrap().folders.store().execute_test_sql(
1747            "CREATE TRIGGER reject_close BEFORE UPDATE OF closed_at ON workflows BEGIN SELECT RAISE(FAIL, 'test close failure'); END;"
1748        );
1749        assert!(
1750            application
1751                .handle_command(
1752                    RequestId(3),
1753                    Command::CloseWorkflow {
1754                        workflow_id: workflow.workflow_id
1755                    }
1756                )
1757                .is_err()
1758        );
1759        let page = application
1760            .workflow_trace(workflow.workflow_id, None, 10)
1761            .unwrap();
1762        assert!(page.spans[0].is_live);
1763        assert_eq!(page.spans[0].status, TraceSpanStatus::Running);
1764        assert_eq!(
1765            application
1766                .trace_events(page.spans[0].span_id, None, 10)
1767                .unwrap()
1768                .events
1769                .len(),
1770            1
1771        );
1772        application
1773            .write_terminal_input(workflow.terminal_id, b"echo alive\n")
1774            .unwrap();
1775        application
1776            .lock_inner()
1777            .unwrap()
1778            .folders
1779            .store()
1780            .execute_test_sql("DROP TRIGGER reject_close");
1781        application
1782            .handle_command(
1783                RequestId(4),
1784                Command::CloseWorkflow {
1785                    workflow_id: workflow.workflow_id,
1786                },
1787            )
1788            .unwrap();
1789        assert_eq!(
1790            application
1791                .workflow_trace(workflow.workflow_id, None, 10)
1792                .unwrap()
1793                .spans[0]
1794                .status,
1795            TraceSpanStatus::Stopped
1796        );
1797    }
1798
1799    #[test]
1800    fn failed_trace_ending_rolls_back_the_workflow_closed_marker() {
1801        let folder = tempfile::tempdir().unwrap();
1802        let application = Application::with_event_capacity(4096).unwrap();
1803        let workflow = create(&application, folder.path());
1804        reject_trace_writes(&application);
1805        assert!(
1806            application
1807                .handle_command(
1808                    RequestId(3),
1809                    Command::CloseWorkflow {
1810                        workflow_id: workflow.workflow_id,
1811                    }
1812                )
1813                .is_err()
1814        );
1815        let stored = application
1816            .lock_inner()
1817            .unwrap()
1818            .folders
1819            .read_store()
1820            .workflows(folder.path())
1821            .unwrap();
1822        assert_eq!(stored.len(), 1);
1823        assert_eq!(stored[0].workflow_id, workflow.workflow_id);
1824        let page = application
1825            .workflow_trace(workflow.workflow_id, None, 10)
1826            .unwrap();
1827        assert!(page.spans[0].is_live);
1828        assert_eq!(page.spans[0].status, TraceSpanStatus::Running);
1829        assert_eq!(
1830            application
1831                .trace_events(page.spans[0].span_id, None, 10)
1832                .unwrap()
1833                .events
1834                .len(),
1835            1
1836        );
1837        assert_eq!(application.snapshot().unwrap().workflows.workflows.len(), 1);
1838        allow_trace_writes(&application);
1839        application
1840            .handle_command(
1841                RequestId(4),
1842                Command::CloseWorkflow {
1843                    workflow_id: workflow.workflow_id,
1844                },
1845            )
1846            .unwrap();
1847        assert!(
1848            application
1849                .lock_inner()
1850                .unwrap()
1851                .folders
1852                .read_store()
1853                .workflows(folder.path())
1854                .unwrap()
1855                .is_empty()
1856        );
1857        assert_eq!(
1858            application
1859                .workflow_trace(workflow.workflow_id, None, 10)
1860                .unwrap()
1861                .spans[0]
1862                .status,
1863            TraceSpanStatus::Stopped
1864        );
1865    }
1866
1867    #[test]
1868    fn durable_trace_anchors_read_the_original_binary_transcript_after_relaunch() {
1869        let data = tempfile::tempdir().unwrap();
1870        let folder = tempfile::tempdir().unwrap();
1871        let (workflow_id, span_id, anchor, expected, events) = {
1872            let mut application =
1873                Application::with_config(data.path(), crate::config::Config::default()).unwrap();
1874            application.terminals.set_test_shell("/bin/sh".into());
1875            let workflow = create(&application, folder.path());
1876            application
1877                .write_terminal_input(
1878                    workflow.terminal_id,
1879                    b"printf '\\377\\000\\033[Htrace bytes\\n'\n",
1880                )
1881                .unwrap();
1882            let deadline = Instant::now() + Duration::from_secs(3);
1883            loop {
1884                while application.next_terminal_chunk().unwrap().is_some() {}
1885                let crate::TranscriptRead::Output(page) = application
1886                    .read_terminal_transcript(
1887                        workflow.terminal_id,
1888                        0,
1889                        crate::MAX_TRANSCRIPT_READ_BYTES,
1890                    )
1891                    .unwrap()
1892                else {
1893                    panic!("new transcript must be retained");
1894                };
1895                if page
1896                    .bytes
1897                    .windows(5)
1898                    .any(|bytes| bytes == [0xff, 0, 0x1b, b'[', b'H'])
1899                {
1900                    break;
1901                }
1902                assert!(
1903                    Instant::now() < deadline,
1904                    "shell did not produce binary output"
1905                );
1906                thread::sleep(Duration::from_millis(5));
1907            }
1908            application
1909                .handle_command(
1910                    RequestId(3),
1911                    Command::CloseWorkflow {
1912                        workflow_id: workflow.workflow_id,
1913                    },
1914                )
1915                .unwrap();
1916            let trace = application
1917                .workflow_trace(workflow.workflow_id, None, 10)
1918                .unwrap();
1919            let span_id = trace.spans[0].span_id;
1920            let events = application.trace_events(span_id, None, 10).unwrap().events;
1921            assert_eq!(events.len(), 2);
1922            let anchor = events[1].anchor.as_ref().unwrap().clone();
1923            assert_eq!(anchor.terminal_id, workflow.terminal_id);
1924            assert!(anchor.byte_offset > 0);
1925            let expected = application
1926                .read_terminal_transcript(anchor.terminal_id, 0, crate::MAX_TRANSCRIPT_READ_BYTES)
1927                .unwrap();
1928            let crate::TranscriptRead::Output(page) = &expected else {
1929                panic!("closed transcript must be retained");
1930            };
1931            assert!(anchor.byte_offset <= page.end_offset);
1932            assert!(
1933                page.bytes
1934                    .windows(5)
1935                    .any(|bytes| bytes == [0xff, 0, 0x1b, b'[', b'H'])
1936            );
1937            (workflow.workflow_id, span_id, anchor, expected, events)
1938        };
1939        let mut application =
1940            Application::with_config(data.path(), crate::config::Config::default()).unwrap();
1941        application.terminals.set_test_shell("/bin/sh".into());
1942        let fresh = create(&application, folder.path());
1943        assert!(fresh.terminal_id.value() > anchor.terminal_id.value());
1944        assert_eq!(
1945            application
1946                .read_terminal_transcript(anchor.terminal_id, 0, crate::MAX_TRANSCRIPT_READ_BYTES)
1947                .unwrap(),
1948            expected
1949        );
1950        let crate::TranscriptRead::Output(page) = application
1951            .read_terminal_transcript(anchor.terminal_id, anchor.byte_offset, 1)
1952            .unwrap()
1953        else {
1954            panic!("trace anchor must remain readable");
1955        };
1956        assert_eq!(page.offset, anchor.byte_offset);
1957        assert_eq!(
1958            application.trace_events(span_id, None, 10).unwrap().events,
1959            events
1960        );
1961        let historical = application.workflow_trace(workflow_id, None, 10).unwrap();
1962        assert_eq!(historical.spans[0].terminal_id, Some(anchor.terminal_id));
1963        assert!(!historical.spans[0].is_live);
1964    }
1965
1966    #[test]
1967    fn observed_output_anchors_survive_queue_drain_and_text_creates_no_events() {
1968        let folder = tempfile::tempdir().unwrap();
1969        let application = Application::with_event_capacity(4096).unwrap();
1970        let workflow = create(&application, folder.path());
1971        application
1972            .write_terminal_input(
1973                workflow.terminal_id,
1974                b"printf 'processExited processStarted\\n'\n",
1975            )
1976            .unwrap();
1977        let deadline = Instant::now() + Duration::from_secs(3);
1978        let mut output = Vec::new();
1979        while Instant::now() < deadline {
1980            while let Some(chunk) = application.next_terminal_chunk().unwrap() {
1981                output.extend(chunk.bytes);
1982            }
1983            if String::from_utf8_lossy(&output).contains("processExited processStarted") {
1984                break;
1985            }
1986            thread::sleep(Duration::from_millis(5));
1987        }
1988        assert!(String::from_utf8_lossy(&output).contains("processExited processStarted"));
1989        let page = application
1990            .workflow_trace(workflow.workflow_id, None, 10)
1991            .unwrap();
1992        let span_id = page.spans[0].span_id;
1993        assert!(page.spans[0].is_live);
1994        assert_eq!(
1995            application
1996                .trace_events(span_id, None, 10)
1997                .unwrap()
1998                .events
1999                .len(),
2000            1
2001        );
2002        let observation = application.terminals.observe(workflow.terminal_id).unwrap();
2003        application
2004            .handle_command(
2005                RequestId(3),
2006                Command::CloseWorkflow {
2007                    workflow_id: workflow.workflow_id,
2008                },
2009            )
2010            .unwrap();
2011        let page = application
2012            .workflow_trace(workflow.workflow_id, None, 10)
2013            .unwrap();
2014        assert_eq!(page.spans[0].status, TraceSpanStatus::Stopped);
2015        assert!(!page.spans[0].is_live);
2016        let events = application.trace_events(span_id, None, 10).unwrap().events;
2017        assert_eq!(events.len(), 2);
2018        assert!(events[1].anchor.as_ref().unwrap().byte_offset >= observation.byte_offset);
2019        assert!(events[1].anchor.as_ref().unwrap().byte_offset > 0);
2020    }
2021
2022    #[test]
2023    fn natural_exit_records_an_exit_without_completing_work_and_close_is_idempotent() {
2024        let folder = tempfile::tempdir().unwrap();
2025        let application = Application::with_event_capacity(4096).unwrap();
2026        let workflow = create(&application, folder.path());
2027        application
2028            .write_terminal_input(workflow.terminal_id, b"exit 7\n")
2029            .unwrap();
2030        let deadline = Instant::now() + Duration::from_secs(3);
2031        while application.snapshot().unwrap().workflows.workflows[0].status
2032            == WorkflowStatus::Running
2033        {
2034            assert!(Instant::now() < deadline, "process exit should be observed");
2035            while application.next_terminal_chunk().unwrap().is_some() {}
2036            thread::sleep(Duration::from_millis(5));
2037        }
2038        let page = application
2039            .workflow_trace(workflow.workflow_id, None, 10)
2040            .unwrap();
2041        let span_id = page.spans[0].span_id;
2042        assert_eq!(page.spans[0].status, TraceSpanStatus::Exited);
2043        application
2044            .handle_command(
2045                RequestId(3),
2046                Command::CloseWorkflow {
2047                    workflow_id: workflow.workflow_id,
2048                },
2049            )
2050            .unwrap();
2051        let events = application.trace_events(span_id, None, 10).unwrap().events;
2052        assert_eq!(events.len(), 2);
2053        assert_eq!(events[1].kind, TraceEventKind::ProcessExited);
2054        assert!(events[1].message.contains("code 7"));
2055    }
2056
2057    #[test]
2058    fn folder_reopen_keeps_history_and_opens_a_distinct_span() {
2059        let folder = tempfile::tempdir().unwrap();
2060        let application = Application::with_event_capacity(4096).unwrap();
2061        let workflow = create(&application, folder.path());
2062        let original = application
2063            .workflow_trace(workflow.workflow_id, None, 10)
2064            .unwrap()
2065            .spans[0]
2066            .span_id;
2067        application
2068            .handle_command(RequestId(3), Command::CloseFolder)
2069            .unwrap();
2070        application
2071            .handle_command(
2072                RequestId(4),
2073                Command::OpenFolder {
2074                    path: folder.path().to_owned(),
2075                },
2076            )
2077            .unwrap();
2078        let page = application
2079            .workflow_trace(workflow.workflow_id, None, 10)
2080            .unwrap();
2081        assert_eq!(page.summary.span_count, 2);
2082        assert_eq!(page.lanes.len(), 1);
2083        assert_ne!(page.spans[0].span_id, original);
2084        assert!(page.spans[0].is_live);
2085        assert_eq!(page.spans[1].span_id, original);
2086        assert_eq!(page.spans[1].status, TraceSpanStatus::Stopped);
2087        assert!(!page.spans[1].is_live);
2088        assert_eq!(application.snapshot().unwrap().traces[0], page.summary);
2089    }
2090}