From 0ba73e0c11548d7b40ff0b91e80b4f99c5dd6788 Mon Sep 17 00:00:00 2001 From: "Luke Boswell (Linux-Desktop)" Date: Fri, 25 Sep 2026 13:02:48 +1000 Subject: [PATCH 1/4] Add binary file reads and buffered seeking --- platform/File.roc | 41 +- platform/Host.roc | 5 + platform/main.roc | 4 + scripts/test_resource_lifetimes.py | 2 +- src/file_reader.rs | 168 +++++ src/lib.rs | 78 ++ src/roc_platform_abi.rs | 704 +++++++++++++++--- .../resource-lifetimes/binary-file-reader.roc | 101 +++ 8 files changed, 984 insertions(+), 119 deletions(-) create mode 100644 src/file_reader.rs create mode 100644 tests/resource-lifetimes/binary-file-reader.roc diff --git a/platform/File.roc b/platform/File.roc index cc2f2a6b..e468e1c4 100644 --- a/platform/File.roc +++ b/platform/File.roc @@ -1,15 +1,20 @@ import Host import Path +import IOErr exposing [IOErr] -## Open files for incremental, buffered reading. +## Read file bytes incrementally and seek within files. ## ## Whole-file operations and filesystem metadata are available on [`Path`](Path). File :: [].{ + ## A byte offset from the start, current logical position, or end of a file. + SeekFrom : [Start(U64), Current(I64), End(I64)] + ## Represents a buffered file reader. ## ## The file is automatically closed when the last reference to the reader is ## dropped. It wraps an opaque host-side `BufReader` handle. + ## Aliases share one cursor, including bytes buffered by line or binary reads. Reader :: { host : Host.FileReader }.{ ## Render the reader without exposing its host handle. @@ -23,6 +28,40 @@ File :: [].{ read_line! = |reader| Host.file_read_line!(reader.host) .map_err(|FileErr(err)| FileErr(err)) + + ## Read at most `max_bytes` bytes. Short reads are valid, even before EOF. + ## For a positive maximum, an empty list means EOF. Zero performs no I/O. + ## Returned bytes remain valid after later reads or seeks. + read_up_to! : Reader, U64 => Try(List(U8), [FileErr(IOErr)]) + read_up_to! = |reader, max_bytes| + Host.file_read_up_to!(reader.host, max_bytes) + .map_err(|FileErr(err)| FileErr(err)) + + ## Read exactly `count` bytes, or report `FileUnexpectedEOF`. + ## Zero succeeds without I/O. Errors may consume bytes; partial data is + ## not returned and the cursor is not restored. + read_exactly! : Reader, U64 => Try(List(U8), [FileErr(IOErr), FileUnexpectedEOF]) + read_exactly! = |reader, count| + match Host.file_read_exactly!(reader.host, count) { + Ok(bytes) => Ok(bytes) + Err(FileUnexpectedEOF) => Err(FileUnexpectedEOF) + Err(FileErr(err)) => Err(FileErr(err)) + } + + ## Return the byte position of the next read, accounting for buffered bytes. + position! : Reader => Try(U64, [FileErr(IOErr)]) + position! = |reader| + Host.file_reader_position!(reader.host) + .map_err(|FileErr(err)| FileErr(err)) + + ## Move the shared cursor and return its new absolute byte position. + ## Seeking past EOF does not extend the file. Invalid offsets and sources + ## that cannot seek report FileErr. A failed seek need not restore the cursor. + ## Aliases of this reader observe the same cursor. + seek! : Reader, SeekFrom => Try(U64, [FileErr(IOErr)]) + seek! = |reader, from| + Host.file_reader_seek!(reader.host, from) + .map_err(|FileErr(err)| FileErr(err)) } ## Open a file for buffered reading using the default buffer capacity. diff --git a/platform/Host.roc b/platform/Host.roc index 53122f27..961a53b2 100644 --- a/platform/Host.roc +++ b/platform/Host.roc @@ -159,4 +159,9 @@ Host :: [].{ tcp_listener_close! : TcpListener => Try({}, Str) env_program_name! : () => Try(NativeOsStr, [ProgramNameUnavailable]) + + file_read_up_to! : FileReader, U64 => Try(List(U8), [FileErr(IOErr)]) + file_read_exactly! : FileReader, U64 => Try(List(U8), [FileErr(IOErr), FileUnexpectedEOF]) + file_reader_position! : FileReader => Try(U64, [FileErr(IOErr)]) + file_reader_seek! : FileReader, [Start(U64), Current(I64), End(I64)] => Try(U64, [FileErr(IOErr)]) } diff --git a/platform/main.roc b/platform/main.roc index 58da87a0..a1fdb974 100644 --- a/platform/main.roc +++ b/platform/main.roc @@ -103,6 +103,10 @@ platform "" "hosted_tcp_accept": Host.tcp_accept!, "hosted_tcp_listener_close": Host.tcp_listener_close!, "hosted_env_program_name": Host.env_program_name!, + "hosted_file_read_up_to": Host.file_read_up_to!, + "hosted_file_read_exactly": Host.file_read_exactly!, + "hosted_file_reader_position": Host.file_reader_position!, + "hosted_file_reader_seek": Host.file_reader_seek!, } targets: { inputs_dir: "targets/", diff --git a/scripts/test_resource_lifetimes.py b/scripts/test_resource_lifetimes.py index 9833f0fa..2a7681fd 100644 --- a/scripts/test_resource_lifetimes.py +++ b/scripts/test_resource_lifetimes.py @@ -48,7 +48,7 @@ def main() -> None: shutil.copy2(source, inputs / name) run("cargo", "build", "--locked", "--lib", "--target", "x86_64-unknown-linux-gnu") shutil.copy2(ROOT / "target/x86_64-unknown-linux-gnu/debug/libhost.a", inputs / "libhost.a") - for name in ("filesystem-tools", "resource-lifetime", "process-control"): + for name in ("filesystem-tools", "resource-lifetime", "process-control", "binary-file-reader"): source = (ROOT / "tests" / "resource-lifetimes" / f"{name}.roc").read_text() source, replacements = re.subn(r'platform "[^"]+"', 'platform "platform/main.roc"', source, count=1) if replacements != 1: diff --git a/src/file_reader.rs b/src/file_reader.rs new file mode 100644 index 00000000..788a0e00 --- /dev/null +++ b/src/file_reader.rs @@ -0,0 +1,168 @@ +//! Binary operations on the same buffered cursor used by line reads. +use std::io::{self, Read, Seek, SeekFrom}; + +fn buffer(count: u64) -> io::Result> { + let size = usize::try_from(count) + .ok() + .filter(|&size| size <= isize::MAX as usize) + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "file read size cannot fit in memory", + ) + })?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(size).map_err(|_| { + io::Error::new( + io::ErrorKind::OutOfMemory, + "could not reserve memory for file read", + ) + })?; + bytes.resize(size, 0); + Ok(bytes) +} + +pub(crate) fn read_up_to(reader: &mut impl Read, count: u64) -> io::Result> { + let mut bytes = buffer(count)?; + if count == 0 { + return Ok(bytes); + } + loop { + match reader.read(&mut bytes) { + Ok(n) => { + bytes.truncate(n); + return Ok(bytes); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + Err(error) => return Err(error), + } + } +} + +pub(crate) fn read_exactly(reader: &mut impl Read, count: u64) -> io::Result> { + let mut bytes = buffer(count)?; + if count != 0 { + reader.read_exact(&mut bytes)?; + } + Ok(bytes) +} + +pub(crate) fn position(reader: &mut impl Seek) -> io::Result { + reader.stream_position() +} + +pub(crate) fn seek(reader: &mut impl Seek, from: SeekFrom) -> io::Result { + reader.seek(from) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{BufRead, BufReader, Cursor}; + + #[test] + fn binary_reads_share_line_buffer_and_logical_position() { + let mut reader = BufReader::with_capacity(8, Cursor::new(b"a\n\0\xff\x80xyz")); + let mut line = Vec::new(); + reader.read_until(b'\n', &mut line).unwrap(); + assert_eq!(line, b"a\n"); + assert_eq!(position(&mut reader).unwrap(), 2); + let retained = read_up_to(&mut reader, 2).unwrap(); + assert_eq!(retained, b"\0\xff"); + assert_eq!(seek(&mut reader, SeekFrom::Current(-1)).unwrap(), 3); + assert_eq!(read_exactly(&mut reader, 3).unwrap(), b"\xff\x80x"); + assert_eq!(position(&mut reader).unwrap(), 6); + assert_eq!(retained, b"\0\xff"); + assert_eq!(seek(&mut reader, SeekFrom::End(-2)).unwrap(), 6); + assert_eq!(read_up_to(&mut reader, 10).unwrap(), b"yz"); + assert!(read_up_to(&mut reader, 10).unwrap().is_empty()); + assert_eq!(seek(&mut reader, SeekFrom::Start(0)).unwrap(), 0); + assert_eq!(read_exactly(&mut reader, 2).unwrap(), b"a\n"); + assert_eq!(seek(&mut reader, SeekFrom::Start(100)).unwrap(), 100); + assert!(read_up_to(&mut reader, 1).unwrap().is_empty()); + assert_eq!(reader.get_ref().get_ref().len(), 8); + assert!(seek(&mut reader, SeekFrom::End(-100)).is_err()); + } + + struct ShortReads { + bytes: Cursor>, + interrupt: bool, + fail: bool, + calls: usize, + } + + #[test] + fn large_binary_input_crosses_buffers_without_losing_bytes() { + let expected: Vec = (0..131_073).map(|i| i as u8).collect(); + let mut reader = BufReader::with_capacity(7, Cursor::new(&expected)); + let mut actual = Vec::new(); + loop { + let chunk = read_up_to(&mut reader, 65_536).unwrap(); + assert!(chunk.len() <= 65_536); + if chunk.is_empty() { + break; + } + actual.extend(chunk); + } + assert_eq!(actual, expected); + seek(&mut reader, SeekFrom::Start(0)).unwrap(); + assert_eq!( + read_exactly(&mut reader, expected.len() as u64).unwrap(), + expected + ); + } + + impl Read for ShortReads { + fn read(&mut self, into: &mut [u8]) -> io::Result { + self.calls += 1; + if std::mem::take(&mut self.interrupt) { + return Err(io::ErrorKind::Interrupted.into()); + } + if self.fail { + return Err(io::ErrorKind::PermissionDenied.into()); + } + let len = into.len().min(2); + self.bytes.read(&mut into[..len]) + } + } + + #[test] + fn short_reads_interruptions_eof_and_errors() { + let mut reader = ShortReads { + bytes: Cursor::new(vec![0, 255, 128, 1, 2, 3, 4]), + interrupt: true, + fail: false, + calls: 0, + }; + assert_eq!(read_up_to(&mut reader, 5).unwrap(), [0, 255]); + assert_eq!(reader.calls, 2); + reader.interrupt = true; + assert_eq!(read_exactly(&mut reader, 4).unwrap(), [128, 1, 2, 3]); + assert_eq!( + read_exactly(&mut reader, 2).unwrap_err().kind(), + io::ErrorKind::UnexpectedEof + ); + assert!(read_up_to(&mut reader, 1).unwrap().is_empty()); + reader.fail = true; + assert_eq!( + read_up_to(&mut reader, 1).unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + assert_eq!( + read_exactly(&mut reader, 1).unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); + let calls = reader.calls; + assert!(read_up_to(&mut reader, 0).unwrap().is_empty()); + assert!(read_exactly(&mut reader, 0).unwrap().is_empty()); + assert_eq!( + read_up_to(&mut reader, u64::MAX).unwrap_err().kind(), + io::ErrorKind::OutOfMemory + ); + assert_eq!( + read_exactly(&mut reader, u64::MAX).unwrap_err().kind(), + io::ErrorKind::OutOfMemory + ); + assert_eq!(reader.calls, calls); + } +} diff --git a/src/lib.rs b/src/lib.rs index c76be0c3..18aafd04 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ mod http; mod roc_platform_abi; mod resources; mod filesystem; +mod file_reader; mod process_service; mod sqlite; mod tcp; @@ -1517,6 +1518,83 @@ pub extern "C" fn hosted_file_read_line(handle: *mut u64) -> FileReaderLineResul result } +#[no_mangle] +pub extern "C" fn hosted_file_read_up_to(handle: *mut u64, count: u64) -> HostFileReadUpToResult { + let host = roc_host(); + let result = match file_reader::read_up_to(unsafe { file_reader_ref(handle) }, count) { + Ok(bytes) => try_file_bytes_ok(roc_u8_list_from_slice(&bytes, host)), + Err(error) => try_file_bytes_err(file_io_err_from_io(&error, host)), + }; + release_file_reader(handle, host); + result +} + +#[no_mangle] +pub extern "C" fn hosted_file_read_exactly(handle: *mut u64, count: u64) -> HostFileReadExactlyResult { + let host = roc_host(); + let result = match file_reader::read_exactly(unsafe { file_reader_ref(handle) }, count) { + Ok(bytes) => HostFileReadExactlyResult { + tag: HostFileReadExactlyResultTag::Ok, + payload: HostFileReadExactlyResultPayload { + ok: ManuallyDrop::new(roc_u8_list_from_slice(&bytes, host)), + }, + }, + Err(error) => { + let error = if error.kind() == io::ErrorKind::UnexpectedEof { + FileErrOrFileUnexpectedEOF { + tag: FileErrOrFileUnexpectedEOFTag::FileUnexpectedEOF, + payload: FileErrOrFileUnexpectedEOFPayload { file_unexpected_eof: [] }, + } + } else { + FileErrOrFileUnexpectedEOF { + tag: FileErrOrFileUnexpectedEOFTag::FileErr, + payload: FileErrOrFileUnexpectedEOFPayload { + file_err: ManuallyDrop::new(file_io_err_from_io(&error, host)), + }, + } + }; + HostFileReadExactlyResult { + tag: HostFileReadExactlyResultTag::Err, + payload: HostFileReadExactlyResultPayload { err: ManuallyDrop::new(error) }, + } + } + }; + release_file_reader(handle, host); + result +} + +#[no_mangle] +pub extern "C" fn hosted_file_reader_position(handle: *mut u64) -> HostFileReaderPositionResult { + let host = roc_host(); + let result = match file_reader::position(unsafe { file_reader_ref(handle) }) { + Ok(position) => try_file_size_ok(position), + Err(error) => try_file_size_err(file_io_err_from_io(&error, host)), + }; + release_file_reader(handle, host); + result +} + +#[no_mangle] +pub extern "C" fn hosted_file_reader_seek( + handle: *mut u64, + from: CurrentOrEndOrStart, +) -> HostFileReaderSeekResult { + let host = roc_host(); + let from = unsafe { + match from.tag { + CurrentOrEndOrStartTag::Start => io::SeekFrom::Start(*from.payload.start), + CurrentOrEndOrStartTag::Current => io::SeekFrom::Current(*from.payload.current), + CurrentOrEndOrStartTag::End => io::SeekFrom::End(*from.payload.end), + } + }; + let result = match file_reader::seek(unsafe { file_reader_ref(handle) }, from) { + Ok(position) => try_file_size_ok(position), + Err(error) => try_file_size_err(file_io_err_from_io(&error, host)), + }; + release_file_reader(handle, host); + result +} + fn file_metadata( path: UnixBytesOrUtf8OrWindowsU16s, roc_host: &RocHost, diff --git a/src/roc_platform_abi.rs b/src/roc_platform_abi.rs index 3cd13418..c5790f8b 100644 --- a/src/roc_platform_abi.rs +++ b/src/roc_platform_abi.rs @@ -7784,6 +7784,389 @@ const _: () = assert!(core::mem::align_of::() == 4, "H #[cfg(target_pointer_width = "32")] const _: () = assert!(core::mem::offset_of!(HostEnvProgramNameResult, tag) == 16, "HostEnvProgramNameResult tag offset mismatch"); +/// Tag discriminant for Try. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostFileReadExactlyResultTag { + Err = 0, + Ok = 1, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub union HostFileReadExactlyResultPayload { + pub err: core::mem::ManuallyDrop, + pub ok: core::mem::ManuallyDrop>, +} + +#[cfg(target_pointer_width = "32")] +#[repr(align(4))] +#[derive(Clone, Copy)] +pub struct HostFileReadExactlyResultPayloadAlignment; + +/// Tag union: Try +#[cfg(target_pointer_width = "32")] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct HostFileReadExactlyResult { + pub _payload_alignment: [HostFileReadExactlyResultPayloadAlignment; 0], + pub payload: [u8; 20], + pub tag: HostFileReadExactlyResultTag, +} + +/// Tag union: Try +#[cfg(not(target_pointer_width = "32"))] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct HostFileReadExactlyResult { + pub payload: HostFileReadExactlyResultPayload, + pub tag: HostFileReadExactlyResultTag, +} + +impl HostFileReadExactlyResult { + /// Borrow the `Err` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Err` and the payload must still be initialized. + #[cfg(target_pointer_width = "32")] + pub unsafe fn borrow_payload_err_unchecked(&self) -> &FileErrOrFileUnexpectedEOF { + unsafe { &*(self.payload.as_ptr() as *const FileErrOrFileUnexpectedEOF) } + } + + /// Borrow the `Err` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Err` and the payload must still be initialized. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn borrow_payload_err_unchecked(&self) -> &FileErrOrFileUnexpectedEOF { + unsafe { &*(&self.payload.err as *const core::mem::ManuallyDrop as *const FileErrOrFileUnexpectedEOF) } + } + + /// Move the `Err` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Err`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(target_pointer_width = "32")] + pub unsafe fn take_payload_err_unchecked(&mut self) -> FileErrOrFileUnexpectedEOF { + unsafe { core::ptr::read(self.payload.as_ptr() as *const FileErrOrFileUnexpectedEOF) } + } + + /// Move the `Err` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Err`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn take_payload_err_unchecked(&mut self) -> FileErrOrFileUnexpectedEOF { + unsafe { core::mem::ManuallyDrop::take(&mut self.payload.err) } + } + + /// Borrow the `Ok` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Ok` and the payload must still be initialized. + #[cfg(target_pointer_width = "32")] + pub unsafe fn borrow_payload_ok_unchecked(&self) -> &RocListWith { + unsafe { &*(self.payload.as_ptr() as *const RocListWith) } + } + + /// Borrow the `Ok` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Ok` and the payload must still be initialized. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn borrow_payload_ok_unchecked(&self) -> &RocListWith { + unsafe { &*(&self.payload.ok as *const core::mem::ManuallyDrop> as *const RocListWith) } + } + + /// Move the `Ok` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Ok`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(target_pointer_width = "32")] + pub unsafe fn take_payload_ok_unchecked(&mut self) -> RocListWith { + unsafe { core::ptr::read(self.payload.as_ptr() as *const RocListWith) } + } + + /// Move the `Ok` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `HostFileReadExactlyResultTag::Ok`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn take_payload_ok_unchecked(&mut self) -> RocListWith { + unsafe { core::mem::ManuallyDrop::take(&mut self.payload.ok) } + } + +} + +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::size_of::() == 48, "HostFileReadExactlyResult size mismatch"); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::align_of::() == 8, "HostFileReadExactlyResult alignment mismatch"); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::offset_of!(HostFileReadExactlyResult, tag) == 40, "HostFileReadExactlyResult tag offset mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::size_of::() == 24, "HostFileReadExactlyResult size mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::align_of::() == 4, "HostFileReadExactlyResult alignment mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::offset_of!(HostFileReadExactlyResult, tag) == 20, "HostFileReadExactlyResult tag offset mismatch"); + +/// Tag discriminant for FileErrOrFileUnexpectedEOF. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileErrOrFileUnexpectedEOFTag { + FileErr = 0, + FileUnexpectedEOF = 1, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub union FileErrOrFileUnexpectedEOFPayload { + pub file_err: core::mem::ManuallyDrop, + pub file_unexpected_eof: [u8; 0], +} + +#[cfg(target_pointer_width = "32")] +#[repr(align(4))] +#[derive(Clone, Copy)] +pub struct FileErrOrFileUnexpectedEOFPayloadAlignment; + +/// Tag union: FileErrOrFileUnexpectedEOF +#[cfg(target_pointer_width = "32")] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct FileErrOrFileUnexpectedEOF { + pub _payload_alignment: [FileErrOrFileUnexpectedEOFPayloadAlignment; 0], + pub payload: [u8; 16], + pub tag: FileErrOrFileUnexpectedEOFTag, +} + +/// Tag union: FileErrOrFileUnexpectedEOF +#[cfg(not(target_pointer_width = "32"))] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct FileErrOrFileUnexpectedEOF { + pub payload: FileErrOrFileUnexpectedEOFPayload, + pub tag: FileErrOrFileUnexpectedEOFTag, +} + +impl FileErrOrFileUnexpectedEOF { + /// Borrow the `FileErr` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `FileErrOrFileUnexpectedEOFTag::FileErr` and the payload must still be initialized. + #[cfg(target_pointer_width = "32")] + pub unsafe fn borrow_payload_file_err_unchecked(&self) -> &IOErr { + unsafe { &*(self.payload.as_ptr() as *const IOErr) } + } + + /// Borrow the `FileErr` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `FileErrOrFileUnexpectedEOFTag::FileErr` and the payload must still be initialized. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn borrow_payload_file_err_unchecked(&self) -> &IOErr { + unsafe { &*(&self.payload.file_err as *const core::mem::ManuallyDrop as *const IOErr) } + } + + /// Move the `FileErr` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `FileErrOrFileUnexpectedEOFTag::FileErr`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(target_pointer_width = "32")] + pub unsafe fn take_payload_file_err_unchecked(&mut self) -> IOErr { + unsafe { core::ptr::read(self.payload.as_ptr() as *const IOErr) } + } + + /// Move the `FileErr` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `FileErrOrFileUnexpectedEOFTag::FileErr`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn take_payload_file_err_unchecked(&mut self) -> IOErr { + unsafe { core::mem::ManuallyDrop::take(&mut self.payload.file_err) } + } + +} + +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::size_of::() == 40, "FileErrOrFileUnexpectedEOF size mismatch"); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::align_of::() == 8, "FileErrOrFileUnexpectedEOF alignment mismatch"); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::offset_of!(FileErrOrFileUnexpectedEOF, tag) == 32, "FileErrOrFileUnexpectedEOF tag offset mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::size_of::() == 20, "FileErrOrFileUnexpectedEOF size mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::align_of::() == 4, "FileErrOrFileUnexpectedEOF alignment mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::offset_of!(FileErrOrFileUnexpectedEOF, tag) == 16, "FileErrOrFileUnexpectedEOF tag offset mismatch"); + +/// Tag discriminant for CurrentOrEndOrStart. +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CurrentOrEndOrStartTag { + Current = 0, + End = 1, + Start = 2, +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub union CurrentOrEndOrStartPayload { + pub current: core::mem::ManuallyDrop, + pub end: core::mem::ManuallyDrop, + pub start: core::mem::ManuallyDrop, +} + +#[cfg(target_pointer_width = "32")] +#[repr(align(8))] +#[derive(Clone, Copy)] +pub struct CurrentOrEndOrStartPayloadAlignment; + +/// Tag union: CurrentOrEndOrStart +#[cfg(target_pointer_width = "32")] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct CurrentOrEndOrStart { + pub _payload_alignment: [CurrentOrEndOrStartPayloadAlignment; 0], + pub payload: [u8; 8], + pub tag: CurrentOrEndOrStartTag, +} + +/// Tag union: CurrentOrEndOrStart +#[cfg(not(target_pointer_width = "32"))] +#[repr(C)] +#[derive(Clone, Copy)] +pub struct CurrentOrEndOrStart { + pub payload: CurrentOrEndOrStartPayload, + pub tag: CurrentOrEndOrStartTag, +} + +impl CurrentOrEndOrStart { + /// Borrow the `Current` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Current` and the payload must still be initialized. + #[cfg(target_pointer_width = "32")] + pub unsafe fn borrow_payload_current_unchecked(&self) -> &i64 { + unsafe { &*(self.payload.as_ptr() as *const i64) } + } + + /// Borrow the `Current` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Current` and the payload must still be initialized. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn borrow_payload_current_unchecked(&self) -> &i64 { + unsafe { &*(&self.payload.current as *const core::mem::ManuallyDrop as *const i64) } + } + + /// Move the `Current` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Current`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(target_pointer_width = "32")] + pub unsafe fn take_payload_current_unchecked(&mut self) -> i64 { + unsafe { core::ptr::read(self.payload.as_ptr() as *const i64) } + } + + /// Move the `Current` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Current`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn take_payload_current_unchecked(&mut self) -> i64 { + unsafe { core::mem::ManuallyDrop::take(&mut self.payload.current) } + } + + /// Borrow the `End` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::End` and the payload must still be initialized. + #[cfg(target_pointer_width = "32")] + pub unsafe fn borrow_payload_end_unchecked(&self) -> &i64 { + unsafe { &*(self.payload.as_ptr() as *const i64) } + } + + /// Borrow the `End` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::End` and the payload must still be initialized. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn borrow_payload_end_unchecked(&self) -> &i64 { + unsafe { &*(&self.payload.end as *const core::mem::ManuallyDrop as *const i64) } + } + + /// Move the `End` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::End`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(target_pointer_width = "32")] + pub unsafe fn take_payload_end_unchecked(&mut self) -> i64 { + unsafe { core::ptr::read(self.payload.as_ptr() as *const i64) } + } + + /// Move the `End` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::End`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn take_payload_end_unchecked(&mut self) -> i64 { + unsafe { core::mem::ManuallyDrop::take(&mut self.payload.end) } + } + + /// Borrow the `Start` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Start` and the payload must still be initialized. + #[cfg(target_pointer_width = "32")] + pub unsafe fn borrow_payload_start_unchecked(&self) -> &u64 { + unsafe { &*(self.payload.as_ptr() as *const u64) } + } + + /// Borrow the `Start` payload without creating another owner. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Start` and the payload must still be initialized. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn borrow_payload_start_unchecked(&self) -> &u64 { + unsafe { &*(&self.payload.start as *const core::mem::ManuallyDrop as *const u64) } + } + + /// Move the `Start` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Start`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(target_pointer_width = "32")] + pub unsafe fn take_payload_start_unchecked(&mut self) -> u64 { + unsafe { core::ptr::read(self.payload.as_ptr() as *const u64) } + } + + /// Move the `Start` payload out of one owned tag-union shell. + /// + /// # Safety + /// `self.tag` must be `CurrentOrEndOrStartTag::Start`. After this call, `self` is logically uninitialized and must not be read or destroyed. + #[cfg(not(target_pointer_width = "32"))] + pub unsafe fn take_payload_start_unchecked(&mut self) -> u64 { + unsafe { core::mem::ManuallyDrop::take(&mut self.payload.start) } + } + +} + +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::size_of::() == 16, "CurrentOrEndOrStart size mismatch"); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::align_of::() == 8, "CurrentOrEndOrStart alignment mismatch"); +#[cfg(target_pointer_width = "64")] +const _: () = assert!(core::mem::offset_of!(CurrentOrEndOrStart, tag) == 8, "CurrentOrEndOrStart tag offset mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::size_of::() == 16, "CurrentOrEndOrStart size mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::align_of::() == 8, "CurrentOrEndOrStart alignment mismatch"); +#[cfg(target_pointer_width = "32")] +const _: () = assert!(core::mem::offset_of!(CurrentOrEndOrStart, tag) == 8, "CurrentOrEndOrStart tag offset mismatch"); + /// Tag discriminant for OsStr. #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -7949,97 +8332,6 @@ const _: () = assert!(core::mem::align_of::() == 4, "OsStr alignment mism #[cfg(target_pointer_width = "32")] const _: () = assert!(core::mem::offset_of!(OsStr, tag) == 12, "OsStr tag offset mismatch"); -/// Tag discriminant for Try. -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TryType267Tag { - Err = 0, - Ok = 1, -} - -#[repr(C)] -#[derive(Clone, Copy)] -pub union TryType267Payload { - pub err: core::mem::ManuallyDrop, - pub ok: [u8; 0], -} - -#[cfg(target_pointer_width = "32")] -#[repr(align(4))] -#[derive(Clone, Copy)] -pub struct TryType267PayloadAlignment; - -/// Tag union: Try -#[cfg(target_pointer_width = "32")] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct TryType267 { - pub _payload_alignment: [TryType267PayloadAlignment; 0], - pub payload: [u8; 4], - pub tag: TryType267Tag, -} - -/// Tag union: Try -#[cfg(not(target_pointer_width = "32"))] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct TryType267 { - pub payload: TryType267Payload, - pub tag: TryType267Tag, -} - -impl TryType267 { - /// Borrow the `Err` payload without creating another owner. - /// - /// # Safety - /// `self.tag` must be `TryType267Tag::Err` and the payload must still be initialized. - #[cfg(target_pointer_width = "32")] - pub unsafe fn borrow_payload_err_unchecked(&self) -> &i32 { - unsafe { &*(self.payload.as_ptr() as *const i32) } - } - - /// Borrow the `Err` payload without creating another owner. - /// - /// # Safety - /// `self.tag` must be `TryType267Tag::Err` and the payload must still be initialized. - #[cfg(not(target_pointer_width = "32"))] - pub unsafe fn borrow_payload_err_unchecked(&self) -> &i32 { - unsafe { &*(&self.payload.err as *const core::mem::ManuallyDrop as *const i32) } - } - - /// Move the `Err` payload out of one owned tag-union shell. - /// - /// # Safety - /// `self.tag` must be `TryType267Tag::Err`. After this call, `self` is logically uninitialized and must not be read or destroyed. - #[cfg(target_pointer_width = "32")] - pub unsafe fn take_payload_err_unchecked(&mut self) -> i32 { - unsafe { core::ptr::read(self.payload.as_ptr() as *const i32) } - } - - /// Move the `Err` payload out of one owned tag-union shell. - /// - /// # Safety - /// `self.tag` must be `TryType267Tag::Err`. After this call, `self` is logically uninitialized and must not be read or destroyed. - #[cfg(not(target_pointer_width = "32"))] - pub unsafe fn take_payload_err_unchecked(&mut self) -> i32 { - unsafe { core::mem::ManuallyDrop::take(&mut self.payload.err) } - } - -} - -#[cfg(target_pointer_width = "64")] -const _: () = assert!(core::mem::size_of::() == 8, "TryType267 size mismatch"); -#[cfg(target_pointer_width = "64")] -const _: () = assert!(core::mem::align_of::() == 4, "TryType267 alignment mismatch"); -#[cfg(target_pointer_width = "64")] -const _: () = assert!(core::mem::offset_of!(TryType267, tag) == 4, "TryType267 tag offset mismatch"); -#[cfg(target_pointer_width = "32")] -const _: () = assert!(core::mem::size_of::() == 8, "TryType267 size mismatch"); -#[cfg(target_pointer_width = "32")] -const _: () = assert!(core::mem::align_of::() == 4, "TryType267 alignment mismatch"); -#[cfg(target_pointer_width = "32")] -const _: () = assert!(core::mem::offset_of!(TryType267, tag) == 4, "TryType267 tag offset mismatch"); - /// Return type record for Host.env_platform! /// Fields ordered by compiler-emitted ABI offsets. #[cfg(target_pointer_width = "32")] @@ -9072,6 +9364,45 @@ pub struct HostTcpListenerCloseArgs { pub arg0: *mut u64, } +/// Arguments for Host.file_read_up_to! +/// Roc signature: Host.FileReader, U64 => Try(List(U8), [FileErr(IOErr)]) +/// Refcounted fields are owned by the hosted function. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct HostFileReadUpToArgs { + pub arg0: *mut u64, + pub arg1: u64, +} + +/// Arguments for Host.file_read_exactly! +/// Roc signature: Host.FileReader, U64 => Try(List(U8), [FileErr(IOErr), FileUnexpectedEOF]) +/// Refcounted fields are owned by the hosted function. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct HostFileReadExactlyArgs { + pub arg0: *mut u64, + pub arg1: u64, +} + +/// Arguments for Host.file_reader_position! +/// Roc signature: Host.FileReader => Try(U64, [FileErr(IOErr)]) +/// Refcounted fields are owned by the hosted function. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct HostFileReaderPositionArgs { + pub arg0: *mut u64, +} + +/// Arguments for Host.file_reader_seek! +/// Roc signature: Host.FileReader, [Current(I64), End(I64), Start(U64)] => Try(U64, [FileErr(IOErr)]) +/// Refcounted fields are owned by the hosted function. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct HostFileReaderSeekArgs { + pub arg0: *mut u64, + pub arg1: CurrentOrEndOrStart, +} + // Platform Type Aliases pub type HostCmdExecExitCodeArg0 = AnonStructB57902ff7f66e961; @@ -9272,6 +9603,18 @@ pub type HostTcpListenerCloseResultTag = HostTcpWriteResultTag; pub type HostEnvProgramNameOk = UnixBytesOrUtf8OrWindowsU16s; pub type HostEnvProgramNameOkPayload = UnixBytesOrUtf8OrWindowsU16sPayload; pub type HostEnvProgramNameOkTag = UnixBytesOrUtf8OrWindowsU16sTag; +pub type HostFileReadUpToResult = HostFileReadBytesResult; +pub type HostFileReadUpToResultPayload = HostFileReadBytesResultPayload; +pub type HostFileReadUpToResultTag = HostFileReadBytesResultTag; +pub type HostFileReadExactlyErr = FileErrOrFileUnexpectedEOF; +pub type HostFileReadExactlyErrPayload = FileErrOrFileUnexpectedEOFPayload; +pub type HostFileReadExactlyErrTag = FileErrOrFileUnexpectedEOFTag; +pub type HostFileReaderPositionResult = HostFileSizeInBytesResult; +pub type HostFileReaderPositionResultPayload = HostFileSizeInBytesResultPayload; +pub type HostFileReaderPositionResultTag = HostFileSizeInBytesResultTag; +pub type HostFileReaderSeekResult = HostFileSizeInBytesResult; +pub type HostFileReaderSeekResultPayload = HostFileSizeInBytesResultPayload; +pub type HostFileReaderSeekResultTag = HostFileSizeInBytesResultTag; pub type MainForHostArg0 = OsStr; pub type MainForHostArg0Payload = OsStrPayload; pub type MainForHostArg0Tag = OsStrTag; @@ -12292,7 +12635,7 @@ unsafe impl RocRelease for HostEnvProgramNameResultRel } } -impl OsStr { +impl HostFileReadExactlyResult { /// Recursively decrement Roc-owned payloads. /// /// # Safety @@ -12301,16 +12644,12 @@ impl OsStr { let mut value = self; let _ = roc_host; match value.tag { - OsStrTag::UnixBytes => { - let payload = unsafe { value.take_payload_unix_bytes_unchecked() }; - unsafe { payload.decref(roc_host); } - }, - OsStrTag::Utf8 => { - let payload = unsafe { value.take_payload_utf8_unchecked() }; + HostFileReadExactlyResultTag::Err => { + let payload = unsafe { value.take_payload_err_unchecked() }; unsafe { payload.decref(roc_host); } }, - OsStrTag::WindowsU16s => { - let payload = unsafe { value.take_payload_windows_u16s_unchecked() }; + HostFileReadExactlyResultTag::Ok => { + let payload = unsafe { value.take_payload_ok_unchecked() }; unsafe { payload.decref(roc_host); } }, } @@ -12325,31 +12664,70 @@ impl OsStr { let value = self; let _ = amount; match value.tag { - OsStrTag::UnixBytes => { - let payload = unsafe { core::ptr::read(value.borrow_payload_unix_bytes_unchecked()) }; + HostFileReadExactlyResultTag::Err => { + let payload = unsafe { core::ptr::read(value.borrow_payload_err_unchecked()) }; unsafe { payload.incref(amount); } }, - OsStrTag::Utf8 => { - let payload = unsafe { core::ptr::read(value.borrow_payload_utf8_unchecked()) }; + HostFileReadExactlyResultTag::Ok => { + let payload = unsafe { core::ptr::read(value.borrow_payload_ok_unchecked()) }; unsafe { payload.incref(amount); } }, - OsStrTag::WindowsU16s => { - let payload = unsafe { core::ptr::read(value.borrow_payload_windows_u16s_unchecked()) }; + } + } +} + +pub struct HostFileReadExactlyResultRelease; + +unsafe impl RocRelease for HostFileReadExactlyResultRelease { + unsafe fn release(value: HostFileReadExactlyResult, roc_host: &RocHost) { + unsafe { value.decref(roc_host); } + } +} + +impl FileErrOrFileUnexpectedEOF { + /// Recursively decrement Roc-owned payloads. + /// + /// # Safety + /// `self` must own one live Roc reference for each refcounted payload. + pub unsafe fn decref(self, roc_host: &RocHost) { + let mut value = self; + let _ = roc_host; + match value.tag { + FileErrOrFileUnexpectedEOFTag::FileErr => { + let payload = unsafe { value.take_payload_file_err_unchecked() }; + unsafe { payload.decref(roc_host); } + }, + FileErrOrFileUnexpectedEOFTag::FileUnexpectedEOF => {}, + } + } + + /// Increment Roc-owned payloads. + /// + /// # Safety + /// `self` must point at live Roc allocations. The retained references must + /// be balanced by later decrefs. + pub unsafe fn incref(self, amount: isize) { + let value = self; + let _ = amount; + match value.tag { + FileErrOrFileUnexpectedEOFTag::FileErr => { + let payload = unsafe { core::ptr::read(value.borrow_payload_file_err_unchecked()) }; unsafe { payload.incref(amount); } }, + FileErrOrFileUnexpectedEOFTag::FileUnexpectedEOF => {}, } } } -pub struct OsStrRelease; +pub struct FileErrOrFileUnexpectedEOFRelease; -unsafe impl RocRelease for OsStrRelease { - unsafe fn release(value: OsStr, roc_host: &RocHost) { +unsafe impl RocRelease for FileErrOrFileUnexpectedEOFRelease { + unsafe fn release(value: FileErrOrFileUnexpectedEOF, roc_host: &RocHost) { unsafe { value.decref(roc_host); } } } -impl TryType267 { +impl CurrentOrEndOrStart { /// Recursively decrement Roc-owned payloads. /// /// # Safety @@ -12358,8 +12736,57 @@ impl TryType267 { let value = self; let _ = roc_host; match value.tag { - TryType267Tag::Err => {}, - TryType267Tag::Ok => {}, + CurrentOrEndOrStartTag::Current => {}, + CurrentOrEndOrStartTag::End => {}, + CurrentOrEndOrStartTag::Start => {}, + } + } + + /// Increment Roc-owned payloads. + /// + /// # Safety + /// `self` must point at live Roc allocations. The retained references must + /// be balanced by later decrefs. + pub unsafe fn incref(self, amount: isize) { + let value = self; + let _ = amount; + match value.tag { + CurrentOrEndOrStartTag::Current => {}, + CurrentOrEndOrStartTag::End => {}, + CurrentOrEndOrStartTag::Start => {}, + } + } +} + +pub struct CurrentOrEndOrStartRelease; + +unsafe impl RocRelease for CurrentOrEndOrStartRelease { + unsafe fn release(value: CurrentOrEndOrStart, roc_host: &RocHost) { + unsafe { value.decref(roc_host); } + } +} + +impl OsStr { + /// Recursively decrement Roc-owned payloads. + /// + /// # Safety + /// `self` must own one live Roc reference for each refcounted payload. + pub unsafe fn decref(self, roc_host: &RocHost) { + let mut value = self; + let _ = roc_host; + match value.tag { + OsStrTag::UnixBytes => { + let payload = unsafe { value.take_payload_unix_bytes_unchecked() }; + unsafe { payload.decref(roc_host); } + }, + OsStrTag::Utf8 => { + let payload = unsafe { value.take_payload_utf8_unchecked() }; + unsafe { payload.decref(roc_host); } + }, + OsStrTag::WindowsU16s => { + let payload = unsafe { value.take_payload_windows_u16s_unchecked() }; + unsafe { payload.decref(roc_host); } + }, } } @@ -12372,16 +12799,26 @@ impl TryType267 { let value = self; let _ = amount; match value.tag { - TryType267Tag::Err => {}, - TryType267Tag::Ok => {}, + OsStrTag::UnixBytes => { + let payload = unsafe { core::ptr::read(value.borrow_payload_unix_bytes_unchecked()) }; + unsafe { payload.incref(amount); } + }, + OsStrTag::Utf8 => { + let payload = unsafe { core::ptr::read(value.borrow_payload_utf8_unchecked()) }; + unsafe { payload.incref(amount); } + }, + OsStrTag::WindowsU16s => { + let payload = unsafe { core::ptr::read(value.borrow_payload_windows_u16s_unchecked()) }; + unsafe { payload.incref(amount); } + }, } } } -pub struct TryType267Release; +pub struct OsStrRelease; -unsafe impl RocRelease for TryType267Release { - unsafe fn release(value: TryType267, roc_host: &RocHost) { +unsafe impl RocRelease for OsStrRelease { + unsafe fn release(value: OsStr, roc_host: &RocHost) { unsafe { value.decref(roc_host); } } } @@ -13122,6 +13559,39 @@ unsafe extern "C" { /// The result is owned by Roc: return exactly one owned reference. pub fn hosted_env_program_name() -> HostEnvProgramNameResult; + /// Hosted symbol for Host.file_read_up_to! + /// Roc signature: Host.FileReader, U64 => Try(List(U8), [FileErr(IOErr)]) + /// Owned arguments. Release each exactly once before returning, unless it is + /// moved into storage or into the result: + /// unsafe { decref_box_with(arg0 as RocBox, core::mem::align_of::(), false, None, roc_host); } + /// The result is owned by Roc: return exactly one owned reference. + pub fn hosted_file_read_up_to(arg0: *mut u64, arg1: u64) -> HostFileReadBytesResult; + + /// Hosted symbol for Host.file_read_exactly! + /// Roc signature: Host.FileReader, U64 => Try(List(U8), [FileErr(IOErr), FileUnexpectedEOF]) + /// Owned arguments. Release each exactly once before returning, unless it is + /// moved into storage or into the result: + /// unsafe { decref_box_with(arg0 as RocBox, core::mem::align_of::(), false, None, roc_host); } + /// The result is owned by Roc: return exactly one owned reference. + pub fn hosted_file_read_exactly(arg0: *mut u64, arg1: u64) -> HostFileReadExactlyResult; + + /// Hosted symbol for Host.file_reader_position! + /// Roc signature: Host.FileReader => Try(U64, [FileErr(IOErr)]) + /// Owned arguments. Release each exactly once before returning, unless it is + /// moved into storage or into the result: + /// unsafe { decref_box_with(arg0 as RocBox, core::mem::align_of::(), false, None, roc_host); } + /// The result is owned by Roc: return exactly one owned reference. + pub fn hosted_file_reader_position(arg0: *mut u64) -> HostFileSizeInBytesResult; + + /// Hosted symbol for Host.file_reader_seek! + /// Roc signature: Host.FileReader, [Current(I64), End(I64), Start(U64)] => Try(U64, [FileErr(IOErr)]) + /// Owned arguments. Release each exactly once before returning, unless it is + /// moved into storage or into the result: + /// unsafe { decref_box_with(arg0 as RocBox, core::mem::align_of::(), false, None, roc_host); } + /// unsafe { arg1.decref(roc_host); } + /// The result is owned by Roc: return exactly one owned reference. + pub fn hosted_file_reader_seek(arg0: *mut u64, arg1: CurrentOrEndOrStart) -> HostFileSizeInBytesResult; + } /// Default memory management functions for Roc platform helpers. diff --git a/tests/resource-lifetimes/binary-file-reader.roc b/tests/resource-lifetimes/binary-file-reader.roc new file mode 100644 index 00000000..7699e4be --- /dev/null +++ b/tests/resource-lifetimes/binary-file-reader.roc @@ -0,0 +1,101 @@ +## Exercise binary reads, buffered seeks, aliases, and cleanup on read failure. +app [main!] { pf: platform "../../platform/main.roc" } + +import pf.Env +import pf.File +import pf.OsStr +import pf.Path +import pf.Stdout + +main! : List(OsStr) => Try({}, _) +main! = |_args| Env.with_temp_dir!( + |workspace| { + path = Path.join(workspace, "binary.dat") + Path.write_bytes!(path, [97, 10, 0, 255, 128, 120, 121, 122])? + check_reader!(path)? + match fail_early!(path) { + Err(FileUnexpectedEOF) => {} + Err(err) => return Err(err) + Ok(_) => return Err(ExpectedTruncation) + } + Path.delete!(path)? + Path.write_bytes!(path, [])? + check_empty!(path)? + Path.delete!(path)? + Stdout.line!("Binary reads, seeks, and resource cleanup passed") + }, +) + +check_reader! = |path| { + reader = File.open_reader_with_capacity!(path, 4)? + alias = reader + line = reader.read_line!()? + expect line == [97, 10] + var $position = alias.position!()? + expect $position == 2 + retained = alias.read_up_to!(2)? + expect retained == [0, 255] + $position = reader.seek!(Current(-1))? + expect $position == 3 + var $bytes = alias.read_exactly!(3)? + expect $bytes == [255, 128, 120] + $position = reader.position!()? + expect $position == 6 + $position = reader.seek!(End(-2))? + expect $position == 6 + $bytes = reader.read_exactly!(2)? + expect $bytes == [121, 122] + $bytes = reader.read_up_to!(1)? + expect $bytes == [] + $position = reader.seek!(Start(0))? + expect $position == 0 + $bytes = reader.read_exactly!(2)? + expect $bytes == [97, 10] + $bytes = reader.read_up_to!(0)? + expect $bytes == [] + $bytes = reader.read_exactly!(0)? + expect $bytes == [] + $position = reader.position!()? + expect $position == 2 + match reader.read_exactly!(U64.highest) { + Err(FileErr(OutOfMemory)) => {} + Err(err) => return Err(err) + Ok(_) => return Err(ExpectedAllocationFailure) + } + match reader.read_up_to!(U64.highest) { + Err(FileErr(OutOfMemory)) => {} + Err(err) => return Err(err) + Ok(_) => return Err(ExpectedAllocationFailure) + } + $position = alias.position!()? + expect $position == 2 + $position = reader.seek!(Start(100))? + expect $position == 100 + $bytes = alias.read_up_to!(1)? + expect $bytes == [] + match reader.seek!(End(-100)) { + Err(FileErr(_)) => {} + Ok(_) => return Err(ExpectedInvalidSeek) + } + $position = reader.seek!(Start(0))? + expect $position == 0 + expect retained == [0, 255] + Ok({}) +} + +fail_early! = |path| { + reader = File.open_reader!(path)? + _ = reader.read_exactly!(9)? + Ok({}) +} + +check_empty! = |path| { + reader = File.open_reader!(path)? + bytes = reader.read_up_to!(16)? + expect bytes == [] + match reader.read_exactly!(1) { + Err(FileUnexpectedEOF) => Ok({}) + Err(err) => Err(err) + Ok(_) => Err(ExpectedTruncation) + } +} From 47d3abc7b126536fc6b93503ef13da8ecda6ccad Mon Sep 17 00:00:00 2001 From: "Luke Boswell (Linux-Desktop)" Date: Fri, 25 Sep 2026 13:06:19 +1000 Subject: [PATCH 2/4] Draft lazy file chunk streams pending Roc Stream.custom --- examples/file-read-binary.todoroc | 39 ++++++ platform/File.roc | 36 ++++++ scripts/test_resource_lifetimes.py | 2 +- .../resource-lifetimes/binary-file-chunks.roc | 121 ++++++++++++++++++ 4 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 examples/file-read-binary.todoroc create mode 100644 tests/resource-lifetimes/binary-file-chunks.roc diff --git a/examples/file-read-binary.todoroc b/examples/file-read-binary.todoroc new file mode 100644 index 00000000..24d0423a --- /dev/null +++ b/examples/file-read-binary.todoroc @@ -0,0 +1,39 @@ +## Count binary file bytes in bounded chunks after inspecting its header. +## TODO(https://github.com/roc-lang/roc/issues/11691): enable when Stream.custom is available. +app [main!] { pf: platform "../platform/main.roc" } + +import pf.File +import pf.OsStr +import pf.Path +import pf.Stdout + +main! : List(OsStr) => Try({}, _) +main! = |args| { + path = match args { + [first, ..] => Path.from_os_str(first) + [] => return Err(MissingPathArgument) + } + reader = File.open_reader!(path)? + # Preview at most 16 bytes; even a short or empty file is valid. + header = reader.read_up_to!(16)? + Stdout.line!("Header bytes: ${Str.inspect(header)}")? + _ = reader.seek!(Start(0))? + var $chunks = reader.chunks(65_536)? + var $total = 0.U64 + while Bool.True { + match Stream.next!($chunks) { + Done => { + break + } + Skip({ rest }) => { + $chunks = rest + } + One({ item, rest }) => { + bytes = item? + $total = $total + bytes.len() + $chunks = rest + } + } + } + Stdout.line!("Read ${$total.to_str()} bytes") +} diff --git a/platform/File.roc b/platform/File.roc index e468e1c4..4620daef 100644 --- a/platform/File.roc +++ b/platform/File.roc @@ -62,6 +62,42 @@ File :: [].{ seek! = |reader, from| Host.file_reader_seek!(reader.host, from) .map_err(|FileErr(err)| FileErr(err)) + + ## Lazily read nonempty chunks of at most `max_bytes` from this reader. + ## Zero is rejected immediately with InvalidChunkSize. Construction does + ## no I/O; each pull reads from the current shared cursor. Chunk boundaries + ## do not correspond to records, newlines, or text encoding boundaries. + ## EOF ends the stream; a read error yields one Err item, then ends. + ## Follow the returned continuation when pulling. To resume after EOF + ## and a seek, create a new stream. Retained chunks remain valid. + ## The stream keeps the file open while it retains a reader. Its terminal + ## continuation holds no reader. Process chunks individually to bound memory; + ## Stream.collect! accumulates every chunk and does not propagate Err items. + chunks : Reader, U64 -> Try(Stream(Try(List(U8), [FileErr(IOErr)])), [InvalidChunkSize]) + chunks = |reader, max_bytes| + if max_bytes == 0 { + Err(InvalidChunkSize) + } else { + # TODO(https://github.com/roc-lang/roc/issues/11691): requires Stream.custom. + # Keep the reader in the state, not the callback capture, so Finished + # can release it even when the terminal continuation is retained. + Ok( + Stream.custom( + Reading(reader), + Unknown, + |state| + match state { + Finished => Err(NoMore) + Reading(current) => + match read_up_to!(current, max_bytes) { + Ok(bytes) if bytes.is_empty() => Err(NoMore) + Ok(bytes) => Ok((Ok(bytes), Reading(current))) + Err(FileErr(err)) => Ok((Err(FileErr(err)), Finished)) + } + }, + ), + ) + } } ## Open a file for buffered reading using the default buffer capacity. diff --git a/scripts/test_resource_lifetimes.py b/scripts/test_resource_lifetimes.py index 2a7681fd..4f4e960c 100644 --- a/scripts/test_resource_lifetimes.py +++ b/scripts/test_resource_lifetimes.py @@ -48,7 +48,7 @@ def main() -> None: shutil.copy2(source, inputs / name) run("cargo", "build", "--locked", "--lib", "--target", "x86_64-unknown-linux-gnu") shutil.copy2(ROOT / "target/x86_64-unknown-linux-gnu/debug/libhost.a", inputs / "libhost.a") - for name in ("filesystem-tools", "resource-lifetime", "process-control", "binary-file-reader"): + for name in ("filesystem-tools", "resource-lifetime", "process-control", "binary-file-reader", "binary-file-chunks"): source = (ROOT / "tests" / "resource-lifetimes" / f"{name}.roc").read_text() source, replacements = re.subn(r'platform "[^"]+"', 'platform "platform/main.roc"', source, count=1) if replacements != 1: diff --git a/tests/resource-lifetimes/binary-file-chunks.roc b/tests/resource-lifetimes/binary-file-chunks.roc new file mode 100644 index 00000000..edd8145d --- /dev/null +++ b/tests/resource-lifetimes/binary-file-chunks.roc @@ -0,0 +1,121 @@ +## TODO(https://github.com/roc-lang/roc/issues/11691): requires Stream.custom. +## Check lazy pulls, terminal errors, shared cursors, and stream-owned handles. +app [main!] { pf: platform "../../platform/main.roc" } + +import pf.Env +import pf.File +import pf.OsStr +import pf.Path +import pf.Stdout + +main! : List(OsStr) => Try({}, _) +main! = |_args| Env.with_temp_dir!( + |workspace| { + path = Path.join(workspace, "chunks.dat") + Path.write_bytes!(path, [0, 255, 128, 10, 1, 2, 3, 4])? + check_chunks!(path)? + for early in [Bool.False, Bool.True] { + match read_one!(path, early) { + Ok(_) => {} + Err(ExpectedEarlyReturn) => {} + Err(err) => return Err(err) + } + } + # This continuation must not keep the file handle alive. + terminal = error_tail!(path)? + Path.delete!(path)? + match Stream.next!(terminal) { + Done => {} + _ => return Err(ExpectedTerminalError) + } + Path.write_bytes!(path, [])? + check_empty!(path)? + Path.delete!(path)? + Stdout.line!("Binary chunk streams and resource cleanup passed") + }, +) + +check_chunks! = |path| { + reader = File.open_reader_with_capacity!(path, 2)? + match reader.chunks(0) { + Err(InvalidChunkSize) => {} + Ok(_) => return Err(ExpectedInvalidChunkSize) + } + stream = reader.chunks(2)? + expect Stream.size_hint(stream) == Unknown + before = reader.position!()? + expect before == 0 + # An alias can move the cursor between construction and consumption. + _ = reader.seek!(Start(1))? + first = pull!(stream)? + expect first.bytes == [255, 128] + after = reader.position!()? + expect after == 3 + # Later pulls consult the same cursor and do not invalidate retained chunks. + _ = reader.seek!(End(-2))? + last = pull!(first.rest)? + expect last.bytes == [3, 4] + match Stream.next!(last.rest) { + Done => {} + _ => return Err(ExpectedEOF) + } + _ = reader.seek!(Start(0))? + # A fresh stream can read again after EOF and a seek. + all = Stream.collect!(reader.chunks(3)?) + var $contents = [] + for result in all { + bytes = result? + expect !bytes.is_empty() + expect bytes.len() <= 3 + $contents = List.concat($contents, bytes) + } + expect $contents == [0, 255, 128, 10, 1, 2, 3, 4] + expect first.bytes == [255, 128] + _ = reader.seek!(Start(0))? + lengths = Stream.collect!(Stream.map(reader.chunks(3)?, |item| item.map_ok(|bytes| bytes.len()))) + var $total = 0.U64 + for length in lengths { + $total = $total + length? + } + expect $total == 8 + Ok({}) +} + +pull! = |stream| + match Stream.next!(stream) { + One({ item, rest }) => Ok({ bytes: item?, rest }) + _ => Err(ExpectedChunk) + } + +read_one! = |path, early| { + # The stream owns the reader after the helper that opened it has returned. + stream = open_chunks!(path, 2)? + first = pull!(stream)? + expect first.bytes == [0, 255] + if early { + return Err(ExpectedEarlyReturn) + } + Ok({}) +} + +error_tail! = |path| { + stream = open_chunks!(path, U64.highest)? + match Stream.next!(stream) { + One({ item: Err(FileErr(OutOfMemory)), rest }) => Ok(rest) + _ => Err(ExpectedReadError) + } +} + +open_chunks! = |path, max_bytes| { + reader = File.open_reader!(path)? + reader.chunks(max_bytes) +} + +check_empty! = |path| { + reader = File.open_reader!(path)? + stream = reader.chunks(8)? + match Stream.next!(stream) { + Done => Ok({}) + _ => Err(ExpectedEOF) + } +} From 4283693e17da8fbf5e04c316af942e1346963ca9 Mon Sep 17 00:00:00 2001 From: "Luke Boswell (Linux-Desktop)" Date: Sun, 27 Sep 2026 19:06:38 +1000 Subject: [PATCH 3/4] Enable binary chunk streams with September 26 Roc nightly --- .github/workflows/ci.yml | 6 +- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/release.yml | 8 +-- ...ad-binary.todoroc => file-read-binary.roc} | 1 - flake.lock | 6 +- flake.nix | 2 +- platform/File.roc | 1 - scripts/test_spec.json | 53 ++++++++++++++++++ tests/fixtures/binary-reader/empty.bin | 0 tests/fixtures/binary-reader/multi-chunk.bin | Bin 0 -> 65543 bytes tests/fixtures/binary-reader/short.bin | Bin 0 -> 8 bytes .../resource-lifetimes/binary-file-chunks.roc | 1 - 12 files changed, 65 insertions(+), 15 deletions(-) rename examples/{file-read-binary.todoroc => file-read-binary.roc} (90%) create mode 100644 tests/fixtures/binary-reader/empty.bin create mode 100644 tests/fixtures/binary-reader/multi-chunk.bin create mode 100644 tests/fixtures/binary-reader/short.bin diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78cb40e6..0b4c6f1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: - uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Install Rust toolchain run: | rustup toolchain install "${{ needs.rust-version.outputs.version }}" --profile minimal --target x86_64-unknown-linux-musl --component llvm-tools-preview @@ -63,7 +63,7 @@ jobs: - uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Install Rust toolchain shell: bash run: | @@ -108,7 +108,7 @@ jobs: - uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Install Rust toolchain shell: bash run: | diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index f19ba7a5..6acab631 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -40,7 +40,7 @@ jobs: uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Restore released docs env: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d55082fb..0e787dca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -114,7 +114,7 @@ jobs: uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Install Zig uses: mlugg/setup-zig@8d6198c65fb0feaa111df26e6b467fea8345e46f @@ -215,7 +215,7 @@ jobs: uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Install Rust shell: bash @@ -260,7 +260,7 @@ jobs: uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Create docs staging tree run: mkdir -p docs-site @@ -432,7 +432,7 @@ jobs: uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b with: version: nightly-new-compiler - nightly-tag: nightly-2026-09-23-c7852fd + nightly-tag: nightly-2026-09-26-d6267b4 - name: Download release metadata uses: actions/download-artifact@v8 diff --git a/examples/file-read-binary.todoroc b/examples/file-read-binary.roc similarity index 90% rename from examples/file-read-binary.todoroc rename to examples/file-read-binary.roc index 24d0423a..63c1e0e5 100644 --- a/examples/file-read-binary.todoroc +++ b/examples/file-read-binary.roc @@ -1,5 +1,4 @@ ## Count binary file bytes in bounded chunks after inspecting its header. -## TODO(https://github.com/roc-lang/roc/issues/11691): enable when Stream.custom is available. app [main!] { pf: platform "../platform/main.roc" } import pf.File diff --git a/flake.lock b/flake.lock index 4b54df90..81e56d24 100644 --- a/flake.lock +++ b/flake.lock @@ -42,11 +42,11 @@ ] }, "locked": { - "lastModified": 1790217927, - "narHash": "sha256-fhatlsqelliHOfoT723TYQb8DzxSn698DpobgiARHIM=", + "lastModified": 1790499958, + "narHash": "sha256-CJCTO4GIwaiJ0eq9XFKOVY5R6pcAfncDpJXtQ65nOG8=", "owner": "roc-lang", "repo": "roc-overlay", - "rev": "06198bdac7c2a171c93d0a6f0ddeea562867ee1e", + "rev": "e2fd3eeabe067747f6e91c59694cd57e93bf098e", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index b5c42a44..f0212dfc 100644 --- a/flake.nix +++ b/flake.nix @@ -75,7 +75,7 @@ default = pkgs.mkShell { packages = [ # Keep in sync with the nightly pinned in .github/workflows. - pkgs.rocpkgs."nightly-2026-09-23-c7852fd" + pkgs.rocpkgs."nightly-2026-09-26-d6267b4" pkgs.python3 rustToolchain pkgs.simple-http-server diff --git a/platform/File.roc b/platform/File.roc index 4620daef..18023147 100644 --- a/platform/File.roc +++ b/platform/File.roc @@ -78,7 +78,6 @@ File :: [].{ if max_bytes == 0 { Err(InvalidChunkSize) } else { - # TODO(https://github.com/roc-lang/roc/issues/11691): requires Stream.custom. # Keep the reader in the state, not the callback capture, so Finished # can release it even when the terminal continuation is retained. Ok( diff --git a/scripts/test_spec.json b/scripts/test_spec.json index 84de8410..87de3c9c 100644 --- a/scripts/test_spec.json +++ b/scripts/test_spec.json @@ -144,6 +144,59 @@ } ] }, + { + "path": "examples/file-read-binary.roc", + "cases": [ + { + "name": "empty", + "args": [ + "{root}/tests/fixtures/binary-reader/empty.bin" + ], + "stdout_contains": [ + "Header bytes: []", + "Read 0 bytes" + ] + }, + { + "name": "short", + "args": [ + "{root}/tests/fixtures/binary-reader/short.bin" + ], + "stdout_contains": [ + "Header bytes: [0, 255, 128, 10, 1, 2, 3, 4]", + "Read 8 bytes" + ] + }, + { + "name": "multi-chunk", + "args": [ + "{root}/tests/fixtures/binary-reader/multi-chunk.bin" + ], + "stdout_contains": [ + "Header bytes: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]", + "Read 65543 bytes" + ] + }, + { + "name": "missing-argument", + "exit_code": 1, + "contains": [ + "MissingPathArgument" + ] + }, + { + "name": "missing-file", + "args": [ + "missing.bin" + ], + "temp_cwd": true, + "exit_code": 1, + "contains": [ + "PathErr(NotFound, Path." + ] + } + ] + }, { "path": "examples/file-read-buffered.roc", "cases": [ diff --git a/tests/fixtures/binary-reader/empty.bin b/tests/fixtures/binary-reader/empty.bin new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/binary-reader/multi-chunk.bin b/tests/fixtures/binary-reader/multi-chunk.bin new file mode 100644 index 0000000000000000000000000000000000000000..056fa96352c057f25e7e51dd9603d0f1b719a0af GIT binary patch literal 65543 zcmeI*g@Tk-7zW_kAY2**>Fx$;>6Gq9P*Op^lkCG-!o+4$c)M?VDOP?WQrp#Hg zX3L%i_I(O;Xt$UB2y?XcQ+pqtCf!_>@96V&` zu;C*{jv75??6~n0CQh3C?Ubq0rq7r;YxbPE^X4yDxM=Z`rOTGDSh;HTnzifJfA{@{ zjhi-a*}84}j-9)9|FCE8zWqNQ`03!GpAR27dhGa#lc!Gq^6PKE|8eH*x$}QsxOnOE zm8*aKeeL>Y|J(k%{eSrf@*m`1$p4UkBL7AHjrliGRia z;-B%~_;>t2{vrR7f64#kpYmV%xBOrJG5?u=&Hv_~^WWb_1p^_W{CEC4|DFHNf9JpR z-}&$Scm6y7o&U~%=fCsc`S1L9{yYDj|IUBszw_Vu@BDZEJO7>k&VT2>^WXXJ{CEC4 z|DFHNf9JpR-}&$Scm6y7o&U~%=fCsc`S1L9{yYDj|IUBszw_Vu@BDZEJO7>k&VT2> z^WXXJ{CEC4|DFHNf9JpR-}&$Scm6y7o&U~%=fCsc`S1L9{yYDj|IUBszw_Vu@BDZE zJO7>k&VT2>^WXXJ{CEC4|DFHNf9JpR-}&$Scm6y7o&U~%=fCsc`S1L9{yYDj|IUBs kzw_Vu@BDZEJO7>k&VT2>2LmCYVd3wy{~nBt`2Q_<2YA4V0RR91 literal 0 HcmV?d00001 diff --git a/tests/fixtures/binary-reader/short.bin b/tests/fixtures/binary-reader/short.bin new file mode 100644 index 0000000000000000000000000000000000000000..37577d806db3a1c9854332bb9f5ac6014ca89b09 GIT binary patch literal 8 PcmZSh-@wJl#LNN!3P%By literal 0 HcmV?d00001 diff --git a/tests/resource-lifetimes/binary-file-chunks.roc b/tests/resource-lifetimes/binary-file-chunks.roc index edd8145d..3d13786d 100644 --- a/tests/resource-lifetimes/binary-file-chunks.roc +++ b/tests/resource-lifetimes/binary-file-chunks.roc @@ -1,4 +1,3 @@ -## TODO(https://github.com/roc-lang/roc/issues/11691): requires Stream.custom. ## Check lazy pulls, terminal errors, shared cursors, and stream-owned handles. app [main!] { pf: platform "../../platform/main.roc" } From 5f99f8280e59bf4fa156bd3936a07daa798ad885 Mon Sep 17 00:00:00 2001 From: "Luke Boswell (Linux-Desktop)" Date: Sun, 27 Sep 2026 19:16:02 +1000 Subject: [PATCH 4/4] Add Nix release packages and bound release API checks --- .github/workflows/ci.yml | 18 +++++- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/release.yml | 43 +++++++++------ CONTRIBUTING.md | 15 +++++ README.md | 14 +++++ flake.nix | 6 +- nix/README.md | 82 +++++++++++++++++++++++++++ nix/consumer.roc | 8 +++ nix/release.json | 11 ++++ nix/release.nix | 54 ++++++++++++++++++ scripts/check_version_bump.py | 88 +++++++++++++++++++++++++++++ scripts/test_check_version_bump.py | 89 ++++++++++++++++++++++++++++++ scripts/test_update_nix_release.py | 49 ++++++++++++++++ scripts/update_nix_release.py | 67 ++++++++++++++++++++++ 14 files changed, 524 insertions(+), 22 deletions(-) create mode 100644 nix/README.md create mode 100644 nix/consumer.roc create mode 100644 nix/release.json create mode 100644 nix/release.nix create mode 100644 scripts/check_version_bump.py create mode 100644 scripts/test_check_version_bump.py create mode 100644 scripts/test_update_nix_release.py create mode 100644 scripts/update_nix_release.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b4c6f1f..4279c3d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,8 +44,8 @@ jobs: run: | rustup toolchain install "${{ needs.rust-version.outputs.version }}" --profile minimal --target x86_64-unknown-linux-musl --component llvm-tools-preview rustup default "${{ needs.rust-version.outputs.version }}" - - name: Test release version validation - run: python3 -m unittest scripts/test_check_release_version.py + - name: Test release tooling + run: python3 -m unittest scripts/test_check_release_version.py scripts/test_check_version_bump.py scripts/test_update_nix_release.py - name: Format, check, and test examples run: ./scripts/test.py --operation validate - name: Check native resource ownership with glibc and Valgrind @@ -54,6 +54,20 @@ jobs: sudo apt-get install --yes valgrind python3 scripts/test_resource_lifetimes.py + nix: + name: Nix release consumer (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, macos-26] + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7 + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 + - name: Build and run an application against the pinned release + run: nix flake check --print-build-logs + validate-windows: name: validate sources (windows) needs: rust-version diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 6acab631..43ae8f04 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -37,7 +37,7 @@ jobs: uses: actions/configure-pages@v6 - name: Install Roc - uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b + uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler nightly-tag: nightly-2026-09-26-d6267b4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e787dca..210d5b4d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,7 @@ jobs: fetch-depth: 0 - name: Install Roc - uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b + uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler nightly-tag: nightly-2026-09-26-d6267b4 @@ -155,18 +155,21 @@ jobs: with: github_token: ${{ github.token }} - # Bootstrap exception: 0.20.0 is a .tar.br bundle in old Roc syntax and - # alpha-0 is also no longer parseable, so current `roc bump` cannot compare - # either predecessor. Change this to `require` after the first stable - # new-compiler .tar.zst release. + # 0.22.2 publicly aliases InternalHttp.TransportErr; roc bump currently + # rejects that alias during API extraction. Keep warning until the compiler + # can compare it, and bound the check so a compiler hang cannot block release. - name: Check version bump - uses: roc-lang/release-package/actions/run-bump-check@af2f0cd8662874b825ff3917f4286e793ba18c55 - with: - release_version: ${{ needs.release-info.outputs.release_version }} - dry_run: ${{ github.event_name != 'workflow_dispatch' }} - bump_check: warn - bump_entrypoint: platform/main.roc - previous_url: ${{ steps.previous.outputs.previous_url }} + env: + RELEASE_VERSION: ${{ needs.release-info.outputs.release_version }} + DRY_RUN: ${{ github.event_name != 'workflow_dispatch' }} + PREVIOUS_URL: ${{ steps.previous.outputs.previous_url }} + run: | + python3 scripts/check_version_bump.py \ + --version "$RELEASE_VERSION" \ + --dry-run "$DRY_RUN" \ + --mode warn \ + --previous-url "$PREVIOUS_URL" \ + --timeout-seconds 120 - name: Bundle platform run: ./scripts/bundle.py --output-dir dist @@ -212,7 +215,7 @@ jobs: uses: actions/checkout@v7 - name: Install Roc - uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b + uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler nightly-tag: nightly-2026-09-26-d6267b4 @@ -257,7 +260,7 @@ jobs: uses: actions/checkout@v7 - name: Install Roc - uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b + uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler nightly-tag: nightly-2026-09-26-d6267b4 @@ -429,7 +432,7 @@ jobs: fetch-depth: 0 - name: Install Roc - uses: roc-lang/setup-roc@bd311e2fb815a3d2255f7ee14a922f0b736e020b + uses: roc-lang/setup-roc@cbe782d6f165b89c87d99f50a59ac4f5f73b4427 # ratchet:roc-lang/setup-roc@v0.3.0 with: version: nightly-new-compiler nightly-tag: nightly-2026-09-26-d6267b4 @@ -440,7 +443,7 @@ jobs: name: release-metadata path: .release - - name: Derive release bundle URL and update examples + - name: Derive release bundle URL and update examples and Nix pins env: RELEASE_VERSION: ${{ needs.build.outputs.release_version }} run: | @@ -463,6 +466,9 @@ jobs: ./scripts/update_app_platform_urls.py \ --platform-url "$bundle_url" \ examples + python3 scripts/update_nix_release.py \ + --version "$RELEASE_VERSION" \ + --url "$bundle_url" - name: Validate updated examples run: | @@ -544,6 +550,7 @@ jobs: Cargo.toml Cargo.lock examples + nix/release.json branch_prefix: release-followup base_branch: ${{ github.event.repository.default_branch }} commit_message: Update sources for ${{ needs.build.outputs.release_version }} @@ -551,6 +558,6 @@ jobs: pr_body: | Release follow-up for ${{ needs.build.outputs.release_version }}. - Updates the Cargo package version and checked-in examples to match - the published platform release. + Updates the Cargo package version, checked-in examples, and Nix + release pins to match the published platform release. github_token: ${{ github.token }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e71e45e2..141984de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -77,6 +77,21 @@ drift independently. When updating the nightly pin in the workflows: ## Verification +For changes to the Nix release outputs, run `nix flake check`. This builds and +runs an application against the pinned published platform inside the Nix sandbox; +see [the Nix consumer documentation](nix/README.md). + +Release-tooling regression tests run with: + +```sh +python3 -m unittest scripts/test_check_release_version.py scripts/test_check_version_bump.py scripts/test_update_nix_release.py +``` + +The release workflow bounds `roc bump` to 120 seconds and saves its output in +`.release/bump-output.txt`. It currently warns on comparison errors and timeouts: +the compiler still rejects the public `InternalHttp.TransportErr` alias in the +0.22.2 predecessor. Switch to `--mode require` once that comparison works. + Run the full local check before opening release or CI-facing changes: ```sh diff --git a/README.md b/README.md index 3ff8f7cb..6e478ae3 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,20 @@ HTTP examples use Roc's builtin `Json` parser and encoder directly through - [`0.21.0-rc4` release documentation](https://roc-lang.github.io/basic-cli/0.21.0-rc4/) - [latest main branch](https://roc-lang.github.io/basic-cli/main/) +## Nix + +Fetch the pinned published platform without rebuilding its Rust hosts: + +```sh +nix build github:roc-lang/basic-cli#platform # result/main.roc and result/targets/ +nix build github:roc-lang/basic-cli#bundle # result is the original release archive +``` + +The default package is `platform`. These outputs provide the release recorded in +[`nix/release.json`](nix/release.json), rather than the current checkout's platform +sources. The flake also provides the compiler and dependency pins needed to build +applications offline; see the [downstream application example](nix/README.md). + ## Help Ask questions on [Roc Zulip](https://roc.zulipchat.com), especially in the `#beginners` stream. diff --git a/flake.nix b/flake.nix index f0212dfc..51a1110b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "basic-cli development environment"; + description = "basic-cli release platform and development environment"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -59,10 +59,14 @@ rust-overlay.overlays.default ]; }; + releaseFor = system: import ./nix/release.nix { pkgs = pkgsFor system; }; in { formatter = forAllSystems (system: (pkgsFor system).nixfmt); + packages = forAllSystems (system: (releaseFor system).packages); + checks = forAllSystems (system: (releaseFor system).checks); + devShells = forAllSystems ( system: let diff --git a/nix/README.md b/nix/README.md new file mode 100644 index 00000000..37f710e8 --- /dev/null +++ b/nix/README.md @@ -0,0 +1,82 @@ +# Using a published basic-cli release with Nix + +`nix build .` and `nix build .#platform` produce an unpacked published platform. +`nix build .#bundle` produces its original `.tar.zst` archive. Both use the URL and +SHA-256 in `release.json`; they do not rebuild this checkout's Rust hosts. The +archive includes every supported platform target. + +The platform imports a separate HTTP package. Its archive is also pinned, so a +consumer can compile inside a Nix sandbox without downloading Roc packages during +the build. `platform.dependencyReplacements` supplies the corresponding compiler +arguments; `platform.compiler` supplies the compiler tested with these packages. + +## Downstream application + +Create this `flake.nix` in your application directory: + +```nix +{ + inputs.basic-cli.url = "github:roc-lang/basic-cli"; + + outputs = { basic-cli, ... }: + let + system = "x86_64-linux"; + pkgs = basic-cli.inputs.nixpkgs.legacyPackages.${system}; + platform = basic-cli.packages.${system}.platform; + in { + packages.${system}.default = pkgs.runCommand "hello" { + nativeBuildInputs = [ platform.compiler ]; + } '' + export HOME="$TMPDIR/home" + mkdir -p "$HOME" "$out/bin" + cp ${pkgs.replaceVars ./main.roc { + platformUrl = platform.release.url; + }} main.roc + roc build main.roc --output=hello \ + --replace-dep '${platform.release.url}' ${platform}/main.roc \ + ${platform.dependencyReplacements} || test "$?" -eq 2 + ./hello + cp hello "$out/bin/hello" + ''; + }; +} +``` + +Create `main.roc` as a template whose platform URL Nix fills in: + +```roc +app [main!] { pf: platform "@platformUrl@" } + +import pf.Stdout + +main! = |_args| Stdout.line!("Hello from basic-cli!") +``` + +Run `nix build` and then `./result/bin/hello`. Commit `flake.lock` to pin the +basic-cli flake and its inputs. For another host, change `system` to +`aarch64-linux` or `aarch64-darwin`; Intel macOS uses `x86_64-darwin` and +`basic-cli.inputs.nixpkgs-x86-darwin` for `pkgs`. + +The `--replace-dep` arguments replace the published URLs with unpacked Nix store +paths. An absolute store path cannot be used directly in a Roc platform header. +The shell accepts Roc exit code 2 because older platform releases can compile +successfully with warnings; running and copying the executable still verifies +that a usable binary was produced. + +The repository's `checks.consumer` builds and runs the same kind of application. +Run `nix flake check` to exercise it locally; CI runs it on Linux and macOS. + +## Updating the release + +The release workflow updates `release.json` in its release follow-up PR alongside +the examples. To update it manually, use Python, GNU tar, and zstd: + +```sh +python3 scripts/update_nix_release.py --version VERSION --url RELEASE_BUNDLE_URL +nix flake check +``` + +The updater hashes the published archive and the direct package downloads named +in its `main.roc`, rather than using the current checkout's package declarations. +If a future dependency introduces its own package imports, extend the pins and +replacement arguments before publishing it through this flake. diff --git a/nix/consumer.roc b/nix/consumer.roc new file mode 100644 index 00000000..97a8e6ee --- /dev/null +++ b/nix/consumer.roc @@ -0,0 +1,8 @@ +app [main!] { pf: platform "@platformUrl@" } + +import pf.Stdout + +main! = |_args| { + Stdout.line!("Hello from basic-cli!")? + Ok({}) +} diff --git a/nix/release.json b/nix/release.json new file mode 100644 index 00000000..8b50b7cd --- /dev/null +++ b/nix/release.json @@ -0,0 +1,11 @@ +{ + "version": "0.22.2", + "url": "https://github.com/roc-lang/basic-cli/releases/download/0.22.2/9zUBxb1LtXYVc4eR4hAtd1WQDwBYDhM6HQdZz1UFCm2m.tar.zst", + "hash": "sha256-5WQ6EAPmGN1R3Y0NSZC1S670PWv5vt22jSQDfsLvEtg=", + "dependencies": { + "http": { + "url": "https://github.com/roc-lang/http/releases/download/1.0.0/6ZUwqYhCS8PU9Mo6MF7oV82ET2o7KYb57CLKDq4cq4sS.tar.zst", + "hash": "sha256-6e+qlQ5y9vds326vAEJFcvppsEumEnMjV6wEU2ePArQ=" + } + } +} diff --git a/nix/release.nix b/nix/release.nix new file mode 100644 index 00000000..d8c33d59 --- /dev/null +++ b/nix/release.nix @@ -0,0 +1,54 @@ +{ pkgs }: +let + release = builtins.fromJSON (builtins.readFile ./release.json); + compiler = pkgs.rocpkgs."nightly-2026-09-26-d6267b4"; + bundle = pkgs.fetchurl { inherit (release) url hash; }; + unpack = + name: archive: + pkgs.runCommand name { nativeBuildInputs = [ pkgs.zstd ]; } '' + mkdir -p "$out" + tar --zstd -xf ${archive} -C "$out" + test -f "$out/main.roc" + ''; + platform = (unpack "basic-cli-${release.version}" bundle).overrideAttrs { + passthru = { + inherit release dependencies compiler; + dependencyReplacements = replacements; + }; + }; + dependencies = builtins.mapAttrs ( + name: dependency: unpack "roc-${name}" (pkgs.fetchurl { inherit (dependency) url hash; }) + ) release.dependencies; + replacements = pkgs.lib.concatStringsSep " " ( + pkgs.lib.mapAttrsToList ( + name: dependency: + "--replace-dep ${pkgs.lib.escapeShellArg dependency.url} ${dependencies.${name}}/main.roc" + ) release.dependencies + ); +in +{ + packages = { + inherit bundle platform; + default = platform; + }; + checks.consumer = + pkgs.runCommand "basic-cli-consumer" + { + nativeBuildInputs = [ platform.compiler ]; + } + '' + export HOME="$TMPDIR/home" + mkdir -p "$HOME" + cp ${pkgs.replaceVars ./consumer.roc { platformUrl = release.url; }} main.roc + # Older releases can produce warnings with the pinned compiler. Roc + # exits 2 after a successful build with warnings; still run the binary. + roc build main.roc --output=hello \ + --replace-dep ${pkgs.lib.escapeShellArg release.url} ${platform}/main.roc \ + ${platform.dependencyReplacements} || test "$?" -eq 2 + ./hello > actual + echo 'Hello from basic-cli!' > expected + diff -u expected actual + mkdir -p "$out/bin" + cp hello "$out/bin/hello" + ''; +} diff --git a/scripts/check_version_bump.py b/scripts/check_version_bump.py new file mode 100644 index 00000000..e8e11f33 --- /dev/null +++ b/scripts/check_version_bump.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Run the release API comparison with a bounded wait and durable diagnostics.""" + +import argparse +import math +import subprocess +import sys +from pathlib import Path + + +def check_bump( + *, + version: str, + previous_url: str, + entrypoint: str, + output_file: Path, + mode: str = "warn", + dry_run: bool = False, + timeout_seconds: float = 120, + roc_command: tuple[str, ...] = ("roc",), +) -> int: + output_file.parent.mkdir(parents=True, exist_ok=True) + if dry_run or mode == "off": + reason = "dry-run release" if dry_run else "bump_check is off" + output_file.write_text(f"roc bump check skipped because {reason}.\n", encoding="utf-8") + return 0 + + failure = None + # The release workflow validates the full version before this step. roc bump + # compares the base semver, including when publishing a prerelease. + base_version = version.split("-", 1)[0].split("+", 1)[0] + with output_file.open("w", encoding="utf-8") as output: + if not previous_url: + failure = "No previous release bundle found; roc bump check skipped." + else: + try: + result = subprocess.run( + [*roc_command, "bump", "--old", previous_url, + "--expect", base_version, entrypoint], + stdout=output, + stderr=subprocess.STDOUT, + timeout=timeout_seconds, + check=False, + ) + if result.returncode != 0: + failure = f"roc bump exited with status {result.returncode}." + except subprocess.TimeoutExpired: + # subprocess.run kills and reaps the compiler before returning. + # Writing straight to the file retains diagnostics from a hang. + failure = f"roc bump timed out after {timeout_seconds:g} seconds." + except OSError as error: + failure = f"Could not run roc bump: {error}" + if failure: + output.write(f"\n{failure}\n") + + diagnostic = output_file.read_text(encoding="utf-8", errors="replace") + if diagnostic: + print(diagnostic, end="" if diagnostic.endswith("\n") else "\n") + if failure: + level = "warning" if mode == "warn" else "error" + print(f"{level}: {failure}", file=sys.stderr) + return 0 if mode == "warn" else 1 + return 0 + + +def positive_seconds(value: str) -> float: + seconds = float(value) + if not math.isfinite(seconds) or seconds <= 0: + raise argparse.ArgumentTypeError("timeout must be a positive finite number") + return seconds + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--previous-url", default="") + parser.add_argument("--entrypoint", default="platform/main.roc") + parser.add_argument("--output-file", type=Path, default=Path(".release/bump-output.txt")) + parser.add_argument("--mode", choices=("warn", "require", "off"), default="warn") + parser.add_argument("--dry-run", choices=("true", "false"), default="false") + parser.add_argument("--timeout-seconds", type=positive_seconds, default=120) + args = vars(parser.parse_args()) + args["dry_run"] = args["dry_run"] == "true" + return check_bump(**args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_check_version_bump.py b/scripts/test_check_version_bump.py new file mode 100644 index 00000000..6d2fb48d --- /dev/null +++ b/scripts/test_check_version_bump.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Exercise release bump handling with real successful, failing and hung children.""" + +import contextlib +import io +import sys +import tempfile +import unittest +from pathlib import Path + +from scripts.check_version_bump import check_bump + + +class CheckVersionBumpTests(unittest.TestCase): + def run_check(self, source: str, **options) -> tuple[int, str, str]: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + compiler = root / "compiler.py" + compiler.write_text(source, encoding="utf-8") + output_file = root / "release" / "bump-output.txt" + stdout, stderr = io.StringIO(), io.StringIO() + arguments = dict( + version="0.23.0-rc1", previous_url="https://example.com/0.22.2/pkg.tar.zst", + entrypoint="platform/main.roc", output_file=output_file, + timeout_seconds=5, roc_command=(sys.executable, str(compiler)), + ) + arguments.update(options) + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + status = check_bump(**arguments) + return status, output_file.read_text(), stdout.getvalue() + stderr.getvalue() + + def test_success_uses_base_version_and_preserves_output(self): + for mode in ("warn", "require"): + with self.subTest(mode=mode): + status, output, log = self.run_check( + "import sys\nassert sys.argv[1:] == ['bump', '--old', " + "'https://example.com/0.22.2/pkg.tar.zst', '--expect', '0.23.0', " + "'platform/main.roc']\nprint('API comparison passed')\n", mode=mode, + ) + self.assertEqual(status, 0) + self.assertIn("API comparison passed", output) + self.assertIn("API comparison passed", log) + + def test_failure_warns_or_requires(self): + for mode, expected in (("warn", 0), ("require", 1)): + with self.subTest(mode=mode): + status, output, log = self.run_check( + "import sys\nprint('extraction failed', file=sys.stderr)\nsys.exit(7)\n", + mode=mode, + ) + self.assertEqual(status, expected) + self.assertIn("extraction failed", output) + self.assertIn("status 7", output) + self.assertIn("warning:" if mode == "warn" else "error:", log) + + def test_timeout_retains_partial_output(self): + for mode, expected in (("warn", 0), ("require", 1)): + with self.subTest(mode=mode): + status, output, log = self.run_check( + "import time\nprint('starting comparison', flush=True)\ntime.sleep(60)\n", + mode=mode, timeout_seconds=1, + ) + self.assertEqual(status, expected) + self.assertIn("starting comparison", output) + self.assertIn("timed out", output) + self.assertIn("timed out", log) + + def test_dry_run_and_off_do_not_start_compiler(self): + for options in ({"dry_run": True, "mode": "require"}, {"mode": "off"}): + with self.subTest(options=options): + status, output, _ = self.run_check("raise AssertionError('must not run')", **options) + self.assertEqual(status, 0) + self.assertTrue(output.startswith("roc bump check skipped")) + + def test_missing_previous_release(self): + for mode, expected in (("warn", 0), ("require", 1)): + status, output, _ = self.run_check("raise AssertionError('must not run')", previous_url="", mode=mode) + self.assertEqual(status, expected) + self.assertIn("No previous release", output) + + def test_missing_compiler_obeys_mode(self): + for mode, expected in (("warn", 0), ("require", 1)): + status, output, _ = self.run_check("", roc_command=("/nonexistent/roc",), mode=mode) + self.assertEqual(status, expected) + self.assertIn("Could not run roc bump", output) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_update_nix_release.py b/scripts/test_update_nix_release.py new file mode 100644 index 00000000..8d3b2d34 --- /dev/null +++ b/scripts/test_update_nix_release.py @@ -0,0 +1,49 @@ +"""Release pins must follow the published archive, not the current checkout.""" +import base64 +import hashlib +import io +import subprocess +import tarfile +import unittest +from unittest.mock import patch + +from scripts.update_nix_release import package_urls, release_metadata + + +class ReleaseMetadataTests(unittest.TestCase): + def test_downloaded_archive_controls_dependencies_and_hashes(self): + platform_url = "https://example.test/platform.tar.zst" + http_url = "https://example.test/http-v2.tar.zst" + source = f'platform "" packages {{ http: "{http_url}", }}'.encode() + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w") as tar: + info = tarfile.TarInfo("main.roc") + info.size = len(source) + tar.addfile(info, io.BytesIO(source)) + compressed = subprocess.check_output(["zstd", "-q", "-c"], input=archive.getvalue()) + downloads = {platform_url: compressed, http_url: b"http package"} + + def open_url(url, timeout): + self.assertGreater(timeout, 0) + return io.BytesIO(downloads[url]) + + with patch("scripts.update_nix_release.urllib.request.urlopen", side_effect=open_url): + metadata = release_metadata("0.24.0", platform_url) + self.assertEqual(metadata["version"], "0.24.0") + self.assertEqual(metadata["url"], platform_url) + for entry in [metadata, metadata["dependencies"]["http"]]: + expected = base64.b64encode(hashlib.sha256(downloads[entry["url"]]).digest()).decode() + self.assertEqual(entry["hash"], "sha256-" + expected) + + def test_unsupported_dependencies_are_not_silently_omitted(self): + with self.assertRaises(ValueError): + package_urls('platform "" packages { local: "../local/main.roc" }') + with self.assertRaises(ValueError): + package_urls('platform ""') + + def test_comments_and_empty_packages(self): + self.assertEqual(package_urls('platform "" packages {\n # no dependencies\n }'), {}) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/update_nix_release.py b/scripts/update_nix_release.py new file mode 100644 index 00000000..5533ba94 --- /dev/null +++ b/scripts/update_nix_release.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Pin a published platform and its package downloads for offline Nix builds.""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +from pathlib import Path +import re +import subprocess +import tempfile +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] + + +def download(url: str, destination: Path) -> str: + digest = hashlib.sha256() + with urllib.request.urlopen(url, timeout=60) as response, destination.open("wb") as output: + while chunk := response.read(1024 * 1024): + output.write(chunk) + digest.update(chunk) + return "sha256-" + base64.b64encode(digest.digest()).decode("ascii") + + +def package_urls(source: str) -> dict[str, str]: + block = re.search(r"\bpackages\s*\{([^}]*)\}", source) + if block is None: + raise ValueError("Published platform has no packages declaration") + declarations = re.sub(r"(?m)^\s*#.*$", "", block[1]) + pattern = r'([A-Za-z_][A-Za-z_0-9]*)\s*:\s*"(https://[^"\s]+\.tar\.zst)"\s*,?' + urls = dict(re.findall(pattern, declarations)) + if re.sub(pattern, "", declarations).strip(): + raise ValueError("Unsupported package declaration; update the Nix dependency pins explicitly") + return urls + + +def release_metadata(version: str, url: str) -> dict[str, object]: + with tempfile.TemporaryDirectory(prefix="basic-cli-nix-release-") as directory: + archive = Path(directory) / "platform.tar.zst" + archive_hash = download(url, archive) + source = subprocess.check_output( + ["tar", "--zstd", "-xOf", str(archive), "main.roc"], text=True, + ) + dependencies = {} + for name, dependency_url in package_urls(source).items(): + dependencies[name] = { + "url": dependency_url, + "hash": download(dependency_url, Path(directory) / f"{name}.tar.zst"), + } + return {"version": version, "url": url, "hash": archive_hash, "dependencies": dependencies} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--version", required=True) + parser.add_argument("--url", required=True) + parser.add_argument("--output", type=Path, default=ROOT / "nix/release.json") + args = parser.parse_args() + metadata = release_metadata(args.version, args.url) + args.output.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + print(f"Pinned basic-cli {args.version} and its package downloads in {args.output}") + + +if __name__ == "__main__": + main()