aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:28:33 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:28:33 +0200
commit91af268aeb40ba7adc69a6e1acf89a94466ecb5e (patch)
tree5077c441b655f7deb160501fafec28697cb0ac90
parenta8a89b3ed3101981d4d87ce50f924ea499c369fd (diff)
downloadele-91af268aeb40ba7adc69a6e1acf89a94466ecb5e.tar.gz
ele-91af268aeb40ba7adc69a6e1acf89a94466ecb5e.zip
Preserve JSONL record boundaries across input files
-rw-r--r--README.md2
-rw-r--r--src/main.rs27
-rw-r--r--tests/cli_tests.rs42
3 files changed, 65 insertions, 6 deletions
diff --git a/README.md b/README.md
index f0d487d..872c0eb 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,7 @@ usage: aip-filter [-p|--print] [--max-record-bytes N] [--] <expr> [file...]
* `--print` prints the parsed expression instead of evaluating it.
* If no files are given, reads from stdin.
-* Input is newline-delimited JSON, with one value per line. Matching lines are preserved.
+* Input is newline-delimited JSON, with one value per line. Matching lines are preserved, with a newline inserted between records when a preceding input file has no trailing newline. A final unterminated record is otherwise left unchanged.
* An empty filter copies the selected input unchanged; `--print` with an empty filter prints nothing.
* Invalid JSON lines are reported and skipped. Any invalid JSON or input/output error causes a nonzero exit status.
* Records are limited to 8 MiB by default, including the newline. `--max-record-bytes N` sets a positive byte limit. Oversized records are reported and skipped without buffering the rest of the line; later records are still processed and the exit status indicates failure. The limit does not apply to an empty filter, which copies bytes directly.
diff --git a/src/main.rs b/src/main.rs
index c93b169..ca62416 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -9,15 +9,31 @@ use std::{
const DEFAULT_MAX_RECORD_BYTES: usize = 8 * 1024 * 1024;
+struct RecordOutput<W> {
+ writer: W,
+ needs_separator: bool,
+}
+
+impl<W: Write> RecordOutput<W> {
+ fn write_record(&mut self, record: &[u8]) -> io::Result<()> {
+ if self.needs_separator {
+ self.writer.write_all(b"\n")?;
+ }
+ self.writer.write_all(record)?;
+ self.needs_separator = !record.ends_with(b"\n");
+ Ok(())
+ }
+}
+
fn filter_lines(
expr: Option<&CompiledFilter<'_>>,
mut reader: impl BufRead,
- out: &mut impl Write,
+ out: &mut RecordOutput<impl Write>,
source: &str,
max_record_bytes: usize,
) -> io::Result<bool> {
let Some(expr) = expr else {
- io::copy(&mut reader, out)?;
+ io::copy(&mut reader, &mut out.writer)?;
return Ok(true);
};
let mut line_buffer = Vec::with_capacity(128);
@@ -38,7 +54,7 @@ fn filter_lines(
match serde_json::from_slice::<Json>(&line_buffer) {
Ok(json) => {
if expr.evaluate(&json) {
- out.write_all(&line_buffer)?;
+ out.write_record(&line_buffer)?;
}
}
Err(e) => {
@@ -97,14 +113,15 @@ fn main() {
}
let filter = expr.as_ref().map(|expr| expr.compile()).transpose()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?;
+ let mut records = RecordOutput { writer: &mut out, needs_separator: false };
if paths.peek().is_none() {
- return filter_lines(filter.as_ref(), io::stdin().lock(), &mut out, "<stdin>", max_record_bytes);
+ return filter_lines(filter.as_ref(), io::stdin().lock(), &mut records, "<stdin>", max_record_bytes);
}
let mut valid = true;
for path in paths {
let file = File::open(Path::new(&path))
.map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
- valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut out, &path, max_record_bytes)
+ valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut records, &path, max_record_bytes)
.map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
}
Ok(valid)
diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs
index 118f139..9e0f848 100644
--- a/tests/cli_tests.rs
+++ b/tests/cli_tests.rs
@@ -239,6 +239,48 @@ fn filtering_multiple_files_preserves_successful_output() {
assert!(String::from_utf8_lossy(&output.stderr).contains("second.jsonl:1: invalid JSON"));
}
+#[test]
+fn multiple_files_keep_record_boundaries_without_forcing_a_final_newline() {
+ let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("target")
+ .join(format!("cli-boundaries-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let first = dir.join("first.jsonl");
+ let middle = dir.join("middle.jsonl");
+ let last = dir.join("last.jsonl");
+ std::fs::write(&middle, b"\n\"unrelated\"\n").unwrap();
+ for ending in ["", "\n", "\r\n"] {
+ std::fs::write(&first, format!("\"match first\"{ending}")).unwrap();
+ std::fs::write(&last, b"\"match last\"").unwrap();
+ let output = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
+ .arg("match")
+ .args([&first, &middle, &last])
+ .output()
+ .unwrap();
+ assert!(output.status.success());
+ let separator = if ending.is_empty() { "\n" } else { ending };
+ assert_eq!(
+ output.stdout,
+ format!("\"match first\"{separator}\"match last\"").as_bytes()
+ );
+ let values: Vec<serde_json::Value> = serde_json::Deserializer::from_slice(&output.stdout)
+ .into_iter()
+ .collect::<Result<_, _>>()
+ .unwrap();
+ assert_eq!(values.len(), 2);
+ }
+ std::fs::write(&first, b"raw").unwrap();
+ std::fs::write(&last, b"bytes").unwrap();
+ let output = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
+ .arg("")
+ .args([&first, &last])
+ .output()
+ .unwrap();
+ std::fs::remove_dir_all(dir).unwrap();
+ assert!(output.status.success());
+ assert_eq!(output.stdout, b"rawbytes");
+}
+
#[cfg(unix)]
#[test]
fn output_flush_errors_are_reported() {