Skip to content

split: restore --io-blocksize & remove io::copy #13643

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
oech3 wants to merge 1 commit into
base: main
Choose a base branch
from
+56 −58
Open

split: restore --io-blocksize & remove io::copy #13643

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 56 additions & 58 deletions src/uu/split/src/split.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,73 +587,71 @@ impl Drop for ByteChunkWriter<'_> {
}
}

impl Write for ByteChunkWriter<'_> {
/// Implements `--bytes=SIZE`
fn write(&mut self, mut buf: &[u8]) -> io::Result<usize> {
impl ByteChunkWriter<'_> {
/// Implements `--bytes=SIZE`. GNU is unbuffered.
// todo: distinct read error and write error and show
fn copy(&mut self, mut reader: impl Read, io_blksize: usize) -> io::Result<()> {
// If the length of `buf` exceeds the number of bytes remaining
// in the current chunk, we will need to write to multiple
// different underlying writers. In that case, each iteration of
// this loop writes to the underlying writer that corresponds to
// the current chunk number.
let mut carryover_bytes_written: usize = 0;
while !buf.is_empty() {
if self.num_bytes_remaining_in_current_chunk == 0 {
// Flush before switching file, so a delayed write error is
// still attributed to the chunk it belongs to.
self.inner.flush().map_err(|e| self.report_error(&e))?;

// Increment the chunk number, reset the number of bytes remaining, and instantiate the new underlying writer.
self.num_chunks_written += 1;
self.num_bytes_remaining_in_current_chunk = self.chunk_size;

// Allocate the new file, since at this point we know there are bytes to be written to it.
let filename = self.filename_iterator.next().ok_or_else(|| {
io::Error::other(translate!("split-error-output-file-suffixes-exhausted"))
})?;
if self.settings.verbose {
print_creating_file(&filename)?;
let mut io_blk = vec![0u8; io_blksize];
while let mut buf = reader.read(&mut io_blk).map(|n| &io_blk[..n])?
&& !buf.is_empty()
{
while !buf.is_empty() {
if self.num_bytes_remaining_in_current_chunk == 0 {
// Flush before switching file, so a delayed write error is
// still attributed to the chunk it belongs to.
self.inner.flush().map_err(|e| self.report_error(&e))?;

// Increment the chunk number, reset the number of bytes remaining, and instantiate the new underlying writer.
self.num_chunks_written += 1;
self.num_bytes_remaining_in_current_chunk = self.chunk_size;

// Allocate the new file, since at this point we know there are bytes to be written to it.
let filename = self.filename_iterator.next().ok_or_else(|| {
io::Error::other(translate!("split-error-output-file-suffixes-exhausted"))
})?;
if self.settings.verbose {
print_creating_file(&filename)?;
}
self.inner = self.settings.instantiate_current_writer(&filename, true)?;
self.current_filename = filename;
self.error_reported = false;
}
self.inner = self.settings.instantiate_current_writer(&filename, true)?;
self.current_filename = filename;
self.error_reported = false;
}

// If the capacity of this chunk is greater than the number of
// bytes in `buf`, then write all the bytes in `buf`. Otherwise,
// write enough bytes to fill the current chunk, then increment
// the chunk number and repeat.
let buf_len = buf.len();
if (buf_len as u64) < self.num_bytes_remaining_in_current_chunk {
let num_bytes_written = self.write_chunk(buf)?;
self.num_bytes_remaining_in_current_chunk -= num_bytes_written as u64;
return Ok(carryover_bytes_written + num_bytes_written);
}
// If the capacity of this chunk is greater than the number of
// bytes in `buf`, then write all the bytes in `buf`. Otherwise,
// write enough bytes to fill the current chunk, then increment
// the chunk number and repeat.
if (buf.len() as u64) < self.num_bytes_remaining_in_current_chunk {
let num_bytes_written = self.write_chunk(buf)?;
self.num_bytes_remaining_in_current_chunk -= num_bytes_written as u64;
break;
}

// Write enough bytes to fill the current chunk.
//
// Conversion to usize is safe because we checked that
// self.num_bytes_remaining_in_current_chunk is lower than
// n, which is already usize.
let i = self.num_bytes_remaining_in_current_chunk as usize;
let num_bytes_written = self.write_chunk(&buf[..i])?;
self.num_bytes_remaining_in_current_chunk -= num_bytes_written as u64;

// It's possible that the underlying writer did not
// write all the bytes.
if num_bytes_written < i {
return Ok(carryover_bytes_written + num_bytes_written);
}
// Write enough bytes to fill the current chunk.
//
// Conversion to usize is safe because we checked that
// self.num_bytes_remaining_in_current_chunk is lower than
// n, which is already usize.
let i = self.num_bytes_remaining_in_current_chunk as usize;
let num_bytes_written = self.write_chunk(&buf[..i])?;
self.num_bytes_remaining_in_current_chunk -= num_bytes_written as u64;

// Move the window to look at only the remaining bytes.
buf = &buf[i..];
// It's possible that the underlying writer did not
// write all the bytes.
if num_bytes_written < i {
break;
}

// Remember for the next iteration that we wrote these bytes.
carryover_bytes_written += num_bytes_written;
// Move the window to look at only the remaining bytes.
buf = &buf[i..];
}
}
Ok(carryover_bytes_written)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush().map_err(|e| self.report_error(&e))
Ok(())
}
}

Expand Down Expand Up @@ -1404,8 +1402,8 @@ fn split(settings: &Settings) -> UResult<()> {
Strategy::Bytes(chunk_size) => {
let mut writer = ByteChunkWriter::new(chunk_size, settings)?;
// todo: distinct read error and write error
match io::copy(&mut reader, &mut writer) {
Ok(_) => Ok(()),
match writer.copy(&mut reader, io_blksize) {
Ok(()) => Ok(()),
// The writer already reported the write error and set the exit
// code, and signals that with an empty error: stay quiet here.
Err(e) if e.kind() == ErrorKind::Other && e.to_string().is_empty() => {
Expand Down
Loading