Skip to main content

twine_core/terminal/
transcript.rs

1use std::collections::VecDeque;
2use std::path::Path;
3use std::sync::atomic::{AtomicBool, Ordering};
4use std::sync::{Arc, Condvar, Mutex, mpsc};
5#[cfg(test)]
6use std::thread;
7use std::thread::JoinHandle;
8
9use thiserror::Error;
10use tracing::warn;
11
12use super::{TerminalChunk, TerminalId, TerminalSize};
13
14mod storage;
15
16use storage::Storage;
17
18/// Maximum bytes returned by a single transcript read.
19pub const MAX_TRANSCRIPT_READ_BYTES: usize = 64 * 1024;
20const MAX_RECORDING_BATCH_BYTES: usize = 64 * 1024;
21
22/// A page of committed, unmodified terminal output. Offsets count bytes, not characters.
23#[derive(Clone, Debug, Eq, PartialEq)]
24pub struct TranscriptPage {
25    pub terminal_id: TerminalId,
26    pub offset: u64,
27    pub next_offset: u64,
28    pub end_offset: u64,
29    pub bytes: Vec<u8>,
30    /// Resize operations preceding bytes in this page, in their original order.
31    pub sizes: Vec<TranscriptSize>,
32    /// Replay needs the complete prefix and its initial geometry.
33    pub replay_available: bool,
34}
35
36#[derive(Clone, Copy, Debug, Eq, PartialEq)]
37pub struct TranscriptSize {
38    pub offset: u64,
39    pub size: TerminalSize,
40}
41
42/// A bounded storage request whose completion can be polled without waiting for disk I/O.
43pub struct TranscriptRequest(mpsc::Receiver<Result<TranscriptRead, TranscriptError>>);
44
45impl TranscriptRequest {
46    /// # Errors
47    /// Returns recording/storage failures, or cancellation when the worker has stopped.
48    pub fn poll(&self) -> Result<Option<TranscriptRead>, TranscriptError> {
49        match self.0.try_recv() {
50            Ok(result) => result.map(Some),
51            Err(mpsc::TryRecvError::Empty) => Ok(None),
52            Err(mpsc::TryRecvError::Disconnected) => Err(TranscriptError::Cancelled),
53        }
54    }
55}
56
57pub(crate) struct Recording {
58    pub chunk: TerminalChunk,
59    pub sizes: Vec<TranscriptSize>,
60    pub geometry_lost: bool,
61}
62
63#[derive(Clone, Debug, Eq, PartialEq)]
64pub enum TranscriptRead {
65    Output(TranscriptPage),
66    /// The requested history was pruned. `None` means no bytes remain for this terminal.
67    Expired {
68        terminal_id: TerminalId,
69        requested_offset: u64,
70        nearest_retained_offset: Option<u64>,
71    },
72}
73
74#[derive(Debug, Error)]
75pub enum TranscriptError {
76    #[error("transcript storage is already owned by another application")]
77    AlreadyOpen,
78    #[error("transcript metadata is corrupt: {0}")]
79    Corrupt(&'static str),
80    #[error("transcript read limit must be between 1 and {MAX_TRANSCRIPT_READ_BYTES} bytes")]
81    InvalidLimit,
82    #[error("invalid transcript storage limits")]
83    InvalidStorageLimits,
84    #[error("transcript identifier or offset overflowed")]
85    Overflow,
86    #[error("terminal {terminal_id:?} has never been allocated")]
87    NotFound { terminal_id: TerminalId },
88    #[error("transcript offset {requested} is beyond the recorded end {end_offset}")]
89    OutOfRange { requested: u64, end_offset: u64 },
90    #[error("transcript operation was cancelled")]
91    Cancelled,
92    #[error("transcript recording is unavailable: {message}")]
93    Unavailable { message: String },
94    #[error(transparent)]
95    Io(#[from] std::io::Error),
96}
97
98#[derive(Clone, Copy, Debug)]
99pub(super) struct Limits {
100    segment_bytes: u64,
101    terminal_bytes: u64,
102    total_bytes: u64,
103    terminal_count: usize,
104    segment_count: usize,
105    pending_bytes: usize,
106    pending_jobs: usize,
107}
108
109impl Default for Limits {
110    fn default() -> Self {
111        Self {
112            segment_bytes: 1024 * 1024,
113            terminal_bytes: 64 * 1024 * 1024,
114            total_bytes: 512 * 1024 * 1024,
115            terminal_count: 1024,
116            segment_count: 512,
117            pending_bytes: 4 * 1024 * 1024,
118            pending_jobs: 256,
119        }
120    }
121}
122
123enum Job {
124    Allocate(mpsc::SyncSender<Result<TerminalId, TranscriptError>>),
125    Append(Recording),
126    Read {
127        terminal_id: TerminalId,
128        offset: u64,
129        limit: usize,
130        reply: mpsc::SyncSender<Result<TranscriptRead, TranscriptError>>,
131    },
132}
133
134impl Job {
135    fn byte_count(&self) -> usize {
136        match self {
137            Self::Append(recording) => recording.chunk.bytes.len() + recording.sizes.len() * 32,
138            Self::Allocate(_) | Self::Read { .. } => 0,
139        }
140    }
141}
142
143struct Queue {
144    jobs: VecDeque<Job>,
145    bytes: usize,
146    accepting: bool,
147    failure: Option<String>,
148}
149
150impl Queue {
151    fn pop_front(&mut self) -> Option<Job> {
152        let mut job = self.jobs.pop_front()?;
153        self.bytes -= job.byte_count();
154        if let Job::Append(recording) = &mut job {
155            let chunk = &mut recording.chunk;
156            // Coalesce only already queued, contiguous output. A read, allocation, or another
157            // terminal remains a FIFO boundary, and the worker never waits to fill a batch.
158            while let Some(Job::Append(next_recording)) = self.jobs.front() {
159                let next = &next_recording.chunk;
160                let end = u64::try_from(chunk.bytes.len())
161                    .ok()
162                    .and_then(|length| chunk.offset.checked_add(length));
163                if chunk.bytes.is_empty()
164                    || recording.geometry_lost
165                    || next_recording.geometry_lost
166                    || next.bytes.is_empty()
167                    || next.terminal_id != chunk.terminal_id
168                    || end != Some(next.offset)
169                    || chunk.bytes.len().saturating_add(next.bytes.len())
170                        > MAX_RECORDING_BATCH_BYTES
171                    || recording.sizes.len() + next_recording.sizes.len() > 256
172                {
173                    break;
174                }
175                let Some(Job::Append(next)) = self.jobs.pop_front() else {
176                    unreachable!("the queue is exclusively owned while taking a batch");
177                };
178                self.bytes -= next.chunk.bytes.len() + next.sizes.len() * 32;
179                chunk.bytes.extend(next.chunk.bytes);
180                recording.sizes.extend(next.sizes);
181                recording.geometry_lost |= next.geometry_lost;
182            }
183        }
184        Some(job)
185    }
186}
187
188struct Shared {
189    queue: Mutex<Queue>,
190    ready: Condvar,
191    space: Condvar,
192    limits: Limits,
193}
194
195pub(crate) struct TranscriptRecorder {
196    shared: Arc<Shared>,
197    worker: Mutex<Option<JoinHandle<()>>>,
198    #[cfg(test)]
199    temporary_directory: Option<tempfile::TempDir>,
200}
201
202impl TranscriptRecorder {
203    #[cfg(test)]
204    pub(crate) fn is_shutting_down(&self) -> bool {
205        !self.shared.queue.lock().unwrap().accepting
206    }
207
208    pub(crate) fn open(path: &Path) -> Result<Self, TranscriptError> {
209        Self::with_limits(path, Limits::default())
210    }
211
212    fn with_limits(path: &Path, limits: Limits) -> Result<Self, TranscriptError> {
213        let storage = Storage::open(path, limits)?;
214        let shared = Arc::new(Shared {
215            queue: Mutex::new(Queue {
216                jobs: VecDeque::new(),
217                bytes: 0,
218                accepting: true,
219                failure: None,
220            }),
221            ready: Condvar::new(),
222            space: Condvar::new(),
223            limits,
224        });
225        let worker_shared = Arc::clone(&shared);
226        let worker = crate::blocking_worker::spawn("terminal-transcripts".into(), move || {
227            run_worker(storage, &worker_shared);
228        })?;
229        Ok(Self {
230            shared,
231            worker: Mutex::new(Some(worker)),
232            #[cfg(test)]
233            temporary_directory: None,
234        })
235    }
236
237    #[cfg(test)]
238    pub(crate) fn temporary() -> Result<Self, TranscriptError> {
239        let directory = tempfile::tempdir()?;
240        let mut recorder = Self::open(directory.path())?;
241        recorder.temporary_directory = Some(directory);
242        Ok(recorder)
243    }
244
245    #[cfg(test)]
246    pub(crate) fn stall_worker(
247        &self,
248        terminal_id: TerminalId,
249    ) -> mpsc::Receiver<Result<TranscriptRead, TranscriptError>> {
250        // A normal read with a rendezvous reply holds the worker until the fixture receives or
251        // drops its response. This exercises real queue backpressure without a fake publish path.
252        let (reply, receiver) = mpsc::sync_channel(0);
253        self.enqueue(
254            Job::Read {
255                terminal_id,
256                offset: 0,
257                limit: 1,
258                reply,
259            },
260            None,
261        )
262        .unwrap();
263        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2);
264        // Each fixture has one stalled read for this terminal. Output may already be queued
265        // behind it, so wait for this request to be taken rather than for global queue emptiness.
266        while self
267            .shared
268            .queue
269            .lock()
270            .unwrap()
271            .jobs
272            .iter()
273            .any(|job| matches!(job, Job::Read { terminal_id: id, .. } if *id == terminal_id))
274        {
275            assert!(
276                std::time::Instant::now() < deadline,
277                "worker did not reach the stalled request"
278            );
279            thread::yield_now();
280        }
281        receiver
282    }
283
284    #[cfg(test)]
285    pub(crate) fn recording_queue_is_full(&self) -> bool {
286        let queue = self.shared.queue.lock().unwrap();
287        queue.jobs.len() >= self.shared.limits.pending_jobs
288            || queue.bytes >= self.shared.limits.pending_bytes
289    }
290
291    #[cfg(test)]
292    pub(crate) fn allocation_is_pending(&self) -> bool {
293        self.shared
294            .queue
295            .lock()
296            .unwrap()
297            .jobs
298            .iter()
299            .any(|job| matches!(job, Job::Allocate(_)))
300    }
301
302    pub(crate) fn allocate(&self) -> Result<TerminalId, TranscriptError> {
303        let (reply, result) = mpsc::sync_channel(1);
304        self.enqueue(Job::Allocate(reply), None)?;
305        result.recv().map_err(disconnected)?
306    }
307
308    #[cfg(test)]
309    pub(super) fn append(
310        &self,
311        chunk: TerminalChunk,
312        cancelled: &AtomicBool,
313    ) -> Result<(), TranscriptError> {
314        self.record(
315            Recording {
316                chunk,
317                sizes: Vec::new(),
318                geometry_lost: false,
319            },
320            cancelled,
321        )
322    }
323
324    pub(super) fn record(
325        &self,
326        recording: Recording,
327        cancelled: &AtomicBool,
328    ) -> Result<(), TranscriptError> {
329        self.enqueue(Job::Append(recording), Some(cancelled))
330    }
331
332    pub(crate) fn read(
333        &self,
334        terminal_id: TerminalId,
335        offset: u64,
336        limit: usize,
337    ) -> Result<TranscriptRead, TranscriptError> {
338        if limit == 0 || limit > MAX_TRANSCRIPT_READ_BYTES {
339            return Err(TranscriptError::InvalidLimit);
340        }
341        let (reply, result) = mpsc::sync_channel(1);
342        self.enqueue(
343            Job::Read {
344                terminal_id,
345                offset,
346                limit,
347                reply,
348            },
349            None,
350        )?;
351        // Reads share the ordered work queue: every previously accepted append is committed first.
352        result.recv().map_err(disconnected)?
353    }
354
355    pub(crate) fn request_read(
356        &self,
357        terminal_id: TerminalId,
358        offset: u64,
359        limit: usize,
360    ) -> Result<Option<TranscriptRequest>, TranscriptError> {
361        if limit == 0 || limit > MAX_TRANSCRIPT_READ_BYTES {
362            return Err(TranscriptError::InvalidLimit);
363        }
364        let mut queue = self.shared.queue.lock().map_err(poisoned)?;
365        if let Some(message) = &queue.failure {
366            return Err(TranscriptError::Unavailable {
367                message: message.clone(),
368            });
369        }
370        if !queue.accepting {
371            return Err(TranscriptError::Cancelled);
372        }
373        if queue.jobs.len() >= self.shared.limits.pending_jobs {
374            return Ok(None);
375        }
376        let (reply, result) = mpsc::sync_channel(1);
377        queue.jobs.push_back(Job::Read {
378            terminal_id,
379            offset,
380            limit,
381            reply,
382        });
383        self.shared.ready.notify_one();
384        Ok(Some(TranscriptRequest(result)))
385    }
386
387    fn enqueue(&self, job: Job, cancelled: Option<&AtomicBool>) -> Result<(), TranscriptError> {
388        let bytes = job.byte_count();
389        if bytes > self.shared.limits.pending_bytes {
390            return Err(TranscriptError::InvalidStorageLimits);
391        }
392        let mut queue = self.shared.queue.lock().map_err(poisoned)?;
393        loop {
394            if cancelled.is_some_and(|cancelled| cancelled.load(Ordering::Acquire)) {
395                return Err(TranscriptError::Cancelled);
396            }
397            if let Some(message) = &queue.failure {
398                return Err(TranscriptError::Unavailable {
399                    message: message.clone(),
400                });
401            }
402            if !queue.accepting {
403                return Err(TranscriptError::Cancelled);
404            }
405            if queue.jobs.len() < self.shared.limits.pending_jobs
406                && bytes <= self.shared.limits.pending_bytes - queue.bytes
407            {
408                queue.bytes += bytes;
409                queue.jobs.push_back(job);
410                self.shared.ready.notify_one();
411                return Ok(());
412            }
413            queue = self.shared.space.wait(queue).map_err(poisoned)?;
414        }
415    }
416
417    pub(super) fn wake_producers(&self) {
418        // Pair with the queue lock so cancellation cannot be lost between checking and waiting.
419        let _queue = self
420            .shared
421            .queue
422            .lock()
423            .unwrap_or_else(std::sync::PoisonError::into_inner);
424        self.shared.space.notify_all();
425    }
426
427    pub(crate) fn shutdown(&self) {
428        {
429            let mut queue = self
430                .shared
431                .queue
432                .lock()
433                .unwrap_or_else(std::sync::PoisonError::into_inner);
434            queue.accepting = false;
435            self.shared.space.notify_all();
436            self.shared.ready.notify_all();
437        }
438        if let Some(worker) = self
439            .worker
440            .lock()
441            .unwrap_or_else(std::sync::PoisonError::into_inner)
442            .take()
443            && worker.join().is_err()
444        {
445            warn!("terminal transcript worker panicked during shutdown");
446        }
447    }
448}
449
450impl Drop for TranscriptRecorder {
451    fn drop(&mut self) {
452        self.shutdown();
453    }
454}
455
456fn run_worker(mut storage: Storage, shared: &Shared) {
457    loop {
458        let (job, failure) = {
459            let mut queue = shared
460                .queue
461                .lock()
462                .unwrap_or_else(std::sync::PoisonError::into_inner);
463            while queue.jobs.is_empty() && queue.accepting {
464                queue = shared
465                    .ready
466                    .wait(queue)
467                    .unwrap_or_else(std::sync::PoisonError::into_inner);
468            }
469            let Some(job) = queue.pop_front() else {
470                return;
471            };
472            shared.space.notify_all();
473            (job, queue.failure.clone())
474        };
475        match job {
476            Job::Allocate(reply) => {
477                let result = failure.map_or_else(|| storage.allocate(), unavailable);
478                if let Err(error) = &result {
479                    record_failure(shared, error);
480                }
481                let _ = reply.send(result);
482            }
483            Job::Append(recording) => {
484                if failure.is_none()
485                    && let Err(error) = storage.append_recording(&recording)
486                {
487                    record_failure(shared, &error);
488                }
489            }
490            Job::Read {
491                terminal_id,
492                offset,
493                limit,
494                reply,
495            } => {
496                let result =
497                    failure.map_or_else(|| storage.read(terminal_id, offset, limit), unavailable);
498                if let Err(error @ (TranscriptError::Io(_) | TranscriptError::Corrupt(_))) = &result
499                {
500                    record_failure(shared, error);
501                }
502                let _ = reply.send(result);
503            }
504        }
505    }
506}
507
508fn record_failure(shared: &Shared, error: &TranscriptError) {
509    let mut queue = shared
510        .queue
511        .lock()
512        .unwrap_or_else(std::sync::PoisonError::into_inner);
513    if queue.failure.is_none() {
514        warn!(%error, "terminal transcript recording failed; live output remains available");
515        queue.failure = Some(error.to_string());
516    }
517    shared.space.notify_all();
518}
519
520fn unavailable<T>(message: String) -> Result<T, TranscriptError> {
521    Err(TranscriptError::Unavailable { message })
522}
523
524fn disconnected(error: mpsc::RecvError) -> TranscriptError {
525    TranscriptError::Unavailable {
526        message: error.to_string(),
527    }
528}
529
530fn poisoned<T>(_: std::sync::PoisonError<T>) -> TranscriptError {
531    TranscriptError::Unavailable {
532        message: "transcript queue lock poisoned".into(),
533    }
534}
535
536#[cfg(test)]
537mod tests {
538    use std::time::Duration;
539
540    use super::*;
541
542    fn output(
543        recorder: &TranscriptRecorder,
544        id: TerminalId,
545        offset: u64,
546        limit: usize,
547    ) -> TranscriptPage {
548        let TranscriptRead::Output(page) = recorder.read(id, offset, limit).unwrap() else {
549            panic!("output should be retained");
550        };
551        page
552    }
553
554    fn append(recorder: &TranscriptRecorder, id: TerminalId, offset: u64, bytes: &[u8]) {
555        recorder
556            .append(
557                TerminalChunk {
558                    terminal_id: id,
559                    offset,
560                    bytes: bytes.to_vec(),
561                },
562                &AtomicBool::new(false),
563            )
564            .unwrap();
565    }
566
567    #[test]
568    fn ordered_reads_flush_accepted_output_and_validate_page_limits() {
569        let recorder = TranscriptRecorder::temporary().unwrap();
570        let id = recorder.allocate().unwrap();
571        append(&recorder, id, 0, &[0xff, 0, 0x1b]);
572        append(&recorder, id, 3, b"[H");
573        assert_eq!(output(&recorder, id, 1, 3).bytes, [0, 0x1b, b'[']);
574        assert_eq!(output(&recorder, id, 4, 3).bytes, b"H");
575        assert!(matches!(
576            recorder.read(id, 0, 0),
577            Err(TranscriptError::InvalidLimit)
578        ));
579        assert!(matches!(
580            recorder.read(id, 0, MAX_TRANSCRIPT_READ_BYTES + 1),
581            Err(TranscriptError::InvalidLimit)
582        ));
583        assert!(matches!(
584            recorder.read(TerminalId::from_value(0), 0, 1),
585            Err(TranscriptError::NotFound { .. })
586        ));
587        assert_eq!(
588            output(&recorder, id, 0, 8).bytes,
589            [0xff, 0, 0x1b, b'[', b'H']
590        );
591    }
592
593    #[test]
594    fn queued_output_preserves_terminal_boundaries_and_ordered_read_ends() {
595        let recorder = Arc::new(TranscriptRecorder::temporary().unwrap());
596        let first = recorder.allocate().unwrap();
597        let second = recorder.allocate().unwrap();
598        let stalled = recorder.stall_worker(first);
599        append(&recorder, first, 0, &[0xff, 0]);
600        append(&recorder, first, 2, &[0x1b, b'[']);
601        append(&recorder, second, 0, b"other");
602        let reading = Arc::clone(&recorder);
603        let reader = thread::spawn(move || output(&reading, first, 0, 8));
604        let deadline = std::time::Instant::now() + Duration::from_secs(2);
605        while !recorder
606            .shared
607            .queue
608            .lock()
609            .unwrap()
610            .jobs
611            .iter()
612            .any(|job| matches!(job, Job::Read { terminal_id, .. } if *terminal_id == first))
613        {
614            assert!(std::time::Instant::now() < deadline, "read was not queued");
615            thread::yield_now();
616        }
617        append(&recorder, first, 4, b"Hlater");
618        drop(stalled);
619        let page = reader.join().unwrap();
620        assert_eq!(page.bytes, [0xff, 0, 0x1b, b'[']);
621        assert_eq!((page.next_offset, page.end_offset), (4, 4));
622        assert_eq!(output(&recorder, first, 4, 8).bytes, b"Hlater");
623        assert_eq!(output(&recorder, second, 0, 8).bytes, b"other");
624    }
625
626    #[test]
627    fn pending_history_requests_do_not_wait_for_the_worker_or_queue_capacity() {
628        let directory = tempfile::tempdir().unwrap();
629        let recorder = TranscriptRecorder::with_limits(
630            directory.path(),
631            Limits {
632                pending_jobs: 2,
633                ..Limits::default()
634            },
635        )
636        .unwrap();
637        let id = recorder.allocate().unwrap();
638        let stalled = recorder.stall_worker(id);
639        let request = recorder.request_read(id, 0, 8).unwrap().unwrap();
640        append(&recorder, id, 0, b"ab");
641        assert!(request.poll().unwrap().is_none());
642        assert!(recorder.request_read(id, 0, 8).unwrap().is_none());
643        drop(stalled);
644        assert_eq!(output(&recorder, id, 0, 8).bytes, b"ab");
645        let Some(TranscriptRead::Output(page)) = request.poll().unwrap() else {
646            panic!("ordered read completes before the later append");
647        };
648        assert_eq!(page.bytes, b"");
649    }
650
651    #[test]
652    fn recording_queue_backpressure_is_released_by_processing_work() {
653        let directory = tempfile::tempdir().unwrap();
654        let limits = Limits {
655            pending_bytes: 2,
656            pending_jobs: 1,
657            ..Limits::default()
658        };
659        let recorder = Arc::new(TranscriptRecorder::with_limits(directory.path(), limits).unwrap());
660        let id = recorder.allocate().unwrap();
661        let stalled = recorder.stall_worker(id);
662        append(&recorder, id, 0, b"ab");
663        let (sent, received) = mpsc::sync_channel(1);
664        let recording = Arc::clone(&recorder);
665        let publisher = thread::spawn(move || {
666            let result = recording.append(
667                TerminalChunk {
668                    terminal_id: id,
669                    offset: 2,
670                    bytes: b"c".to_vec(),
671                },
672                &AtomicBool::new(false),
673            );
674            sent.send(result).unwrap();
675        });
676        assert!(matches!(
677            received.recv_timeout(Duration::from_millis(50)),
678            Err(mpsc::RecvTimeoutError::Timeout)
679        ));
680        stalled
681            .recv_timeout(Duration::from_secs(2))
682            .unwrap()
683            .unwrap();
684        received
685            .recv_timeout(Duration::from_secs(2))
686            .unwrap()
687            .unwrap();
688        publisher.join().unwrap();
689        assert_eq!(output(&recorder, id, 0, 8).bytes, b"abc");
690    }
691
692    #[test]
693    fn cancellation_wakes_a_recording_producer_without_discarding_accepted_output() {
694        let directory = tempfile::tempdir().unwrap();
695        let recorder = Arc::new(
696            TranscriptRecorder::with_limits(
697                directory.path(),
698                Limits {
699                    pending_bytes: 2,
700                    pending_jobs: 1,
701                    ..Limits::default()
702                },
703            )
704            .unwrap(),
705        );
706        let id = recorder.allocate().unwrap();
707        let stalled = recorder.stall_worker(id);
708        append(&recorder, id, 0, b"ab");
709        let cancelled = Arc::new(AtomicBool::new(false));
710        let cancellation = Arc::clone(&cancelled);
711        let recording = Arc::clone(&recorder);
712        let (sent, received) = mpsc::sync_channel(1);
713        let publisher = thread::spawn(move || {
714            sent.send(recording.append(
715                TerminalChunk {
716                    terminal_id: id,
717                    offset: 2,
718                    bytes: b"c".to_vec(),
719                },
720                &cancellation,
721            ))
722            .unwrap();
723        });
724        assert!(matches!(
725            received.recv_timeout(Duration::from_millis(50)),
726            Err(mpsc::RecvTimeoutError::Timeout)
727        ));
728        cancelled.store(true, Ordering::Release);
729        recorder.wake_producers();
730        assert!(matches!(
731            received.recv_timeout(Duration::from_secs(2)).unwrap(),
732            Err(TranscriptError::Cancelled)
733        ));
734        publisher.join().unwrap();
735        drop(stalled);
736        assert_eq!(output(&recorder, id, 0, 8).bytes, b"ab");
737    }
738
739    #[test]
740    fn orderly_shutdown_flushes_the_queue_and_releases_the_directory() {
741        let directory = tempfile::tempdir().unwrap();
742        let recorder = TranscriptRecorder::open(directory.path()).unwrap();
743        let id = recorder.allocate().unwrap();
744        append(&recorder, id, 0, b"pending");
745        recorder.shutdown();
746        let reopened = TranscriptRecorder::open(directory.path()).unwrap();
747        assert_eq!(output(&reopened, id, 0, 8).bytes, b"pending");
748        assert!(reopened.allocate().unwrap().value() > id.value());
749        assert!(matches!(
750            recorder.read(id, 0, 1),
751            Err(TranscriptError::Cancelled)
752        ));
753    }
754
755    #[test]
756    fn storage_failures_are_explicit_and_release_waiting_producers() {
757        let directory = tempfile::tempdir().unwrap();
758        let recorder = Arc::new(
759            TranscriptRecorder::with_limits(
760                directory.path(),
761                Limits {
762                    pending_bytes: 2,
763                    pending_jobs: 1,
764                    ..Limits::default()
765                },
766            )
767            .unwrap(),
768        );
769        let id = recorder.allocate().unwrap();
770        let stalled = recorder.stall_worker(id);
771        std::fs::create_dir(directory.path().join("metadata.next")).unwrap();
772        append(&recorder, id, 0, b"ab");
773        let (sent, received) = mpsc::sync_channel(1);
774        let recording = Arc::clone(&recorder);
775        let publisher = thread::spawn(move || {
776            let result = recording.append(
777                TerminalChunk {
778                    terminal_id: id,
779                    offset: 2,
780                    bytes: b"cd".to_vec(),
781                },
782                &AtomicBool::new(false),
783            );
784            // It may be admitted just before the worker reports failure; its ordered read must
785            // still report failure instead of claiming those bytes were recorded.
786            sent.send(result.and_then(|()| recording.read(id, 0, 8)))
787                .unwrap();
788        });
789        assert!(matches!(
790            received.recv_timeout(Duration::from_millis(50)),
791            Err(mpsc::RecvTimeoutError::Timeout)
792        ));
793        drop(stalled);
794        assert!(matches!(
795            received.recv_timeout(Duration::from_secs(2)).unwrap(),
796            Err(TranscriptError::Unavailable { .. })
797        ));
798        publisher.join().unwrap();
799        assert!(matches!(
800            recorder.read(id, 0, 8),
801            Err(TranscriptError::Unavailable { .. })
802        ));
803        assert!(matches!(
804            recorder.allocate(),
805            Err(TranscriptError::Unavailable { .. })
806        ));
807    }
808}