diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:28:33 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:28:33 +0200 |
| commit | 91af268aeb40ba7adc69a6e1acf89a94466ecb5e (patch) | |
| tree | 5077c441b655f7deb160501fafec28697cb0ac90 | |
| parent | a8a89b3ed3101981d4d87ce50f924ea499c369fd (diff) | |
| download | ele-91af268aeb40ba7adc69a6e1acf89a94466ecb5e.tar.gz ele-91af268aeb40ba7adc69a6e1acf89a94466ecb5e.zip | |
Preserve JSONL record boundaries across input files
| -rw-r--r-- | README.md | 2 | ||||
| -rw-r--r-- | src/main.rs | 27 | ||||
| -rw-r--r-- | tests/cli_tests.rs | 42 |
3 files changed, 65 insertions, 6 deletions
@@ -18,7 +18,7 @@ usage: aip-filter [-p|--print] [--max-record-bytes N] [--] <expr> [file...] * `--print` prints the parsed expression instead of evaluating it. * If no files are given, reads from stdin. -* Input is newline-delimited JSON, with one value per line. Matching lines are preserved. +* Input is newline-delimited JSON, with one value per line. Matching lines are preserved, with a newline inserted between records when a preceding input file has no trailing newline. A final unterminated record is otherwise left unchanged. * An empty filter copies the selected input unchanged; `--print` with an empty filter prints nothing. * Invalid JSON lines are reported and skipped. Any invalid JSON or input/output error causes a nonzero exit status. * Records are limited to 8 MiB by default, including the newline. `--max-record-bytes N` sets a positive byte limit. Oversized records are reported and skipped without buffering the rest of the line; later records are still processed and the exit status indicates failure. The limit does not apply to an empty filter, which copies bytes directly. diff --git a/src/main.rs b/src/main.rs index c93b169..ca62416 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,15 +9,31 @@ use std::{ const DEFAULT_MAX_RECORD_BYTES: usize = 8 * 1024 * 1024; +struct RecordOutput<W> { + writer: W, + needs_separator: bool, +} + +impl<W: Write> RecordOutput<W> { + fn write_record(&mut self, record: &[u8]) -> io::Result<()> { + if self.needs_separator { + self.writer.write_all(b"\n")?; + } + self.writer.write_all(record)?; + self.needs_separator = !record.ends_with(b"\n"); + Ok(()) + } +} + fn filter_lines( expr: Option<&CompiledFilter<'_>>, mut reader: impl BufRead, - out: &mut impl Write, + out: &mut RecordOutput<impl Write>, source: &str, max_record_bytes: usize, ) -> io::Result<bool> { let Some(expr) = expr else { - io::copy(&mut reader, out)?; + io::copy(&mut reader, &mut out.writer)?; return Ok(true); }; let mut line_buffer = Vec::with_capacity(128); @@ -38,7 +54,7 @@ fn filter_lines( match serde_json::from_slice::<Json>(&line_buffer) { Ok(json) => { if expr.evaluate(&json) { - out.write_all(&line_buffer)?; + out.write_record(&line_buffer)?; } } Err(e) => { @@ -97,14 +113,15 @@ fn main() { } let filter = expr.as_ref().map(|expr| expr.compile()).transpose() .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; + let mut records = RecordOutput { writer: &mut out, needs_separator: false }; if paths.peek().is_none() { - return filter_lines(filter.as_ref(), io::stdin().lock(), &mut out, "<stdin>", max_record_bytes); + return filter_lines(filter.as_ref(), io::stdin().lock(), &mut records, "<stdin>", max_record_bytes); } let mut valid = true; for path in paths { let file = File::open(Path::new(&path)) .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?; - valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut out, &path, max_record_bytes) + valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut records, &path, max_record_bytes) .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?; } Ok(valid) diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 118f139..9e0f848 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -239,6 +239,48 @@ fn filtering_multiple_files_preserves_successful_output() { assert!(String::from_utf8_lossy(&output.stderr).contains("second.jsonl:1: invalid JSON")); } +#[test] +fn multiple_files_keep_record_boundaries_without_forcing_a_final_newline() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join(format!("cli-boundaries-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let first = dir.join("first.jsonl"); + let middle = dir.join("middle.jsonl"); + let last = dir.join("last.jsonl"); + std::fs::write(&middle, b"\n\"unrelated\"\n").unwrap(); + for ending in ["", "\n", "\r\n"] { + std::fs::write(&first, format!("\"match first\"{ending}")).unwrap(); + std::fs::write(&last, b"\"match last\"").unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_aip-filter")) + .arg("match") + .args([&first, &middle, &last]) + .output() + .unwrap(); + assert!(output.status.success()); + let separator = if ending.is_empty() { "\n" } else { ending }; + assert_eq!( + output.stdout, + format!("\"match first\"{separator}\"match last\"").as_bytes() + ); + let values: Vec<serde_json::Value> = serde_json::Deserializer::from_slice(&output.stdout) + .into_iter() + .collect::<Result<_, _>>() + .unwrap(); + assert_eq!(values.len(), 2); + } + std::fs::write(&first, b"raw").unwrap(); + std::fs::write(&last, b"bytes").unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_aip-filter")) + .arg("") + .args([&first, &last]) + .output() + .unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, b"rawbytes"); +} + #[cfg(unix)] #[test] fn output_flush_errors_are_reported() { |
