diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:06:34 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:06:34 +0200 |
| commit | 171c32577ef504ec3b8ea09bb8df00fd6be7a13d (patch) | |
| tree | 30844eb0884a36fbabd9e7cb45e6226d139577df | |
| parent | e3eb58307419a9f7314e061c393d579fd14b75ab (diff) | |
| download | ele-171c32577ef504ec3b8ea09bb8df00fd6be7a13d.tar.gz ele-171c32577ef504ec3b8ea09bb8df00fd6be7a13d.zip | |
Bound JSON record reads and report oversized input
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | src/main.rs | 77 | ||||
| -rw-r--r-- | tests/cli_tests.rs | 88 |
3 files changed, 128 insertions, 41 deletions
@@ -13,7 +13,7 @@ cargo install --path . ## CLI usage ```bash -usage: aip-filter [-p|--print] <expr> [file...] +usage: aip-filter [-p|--print] [--max-record-bytes N] [--] <expr> [file...] ``` * `--print` prints the parsed expression instead of evaluating it. @@ -21,6 +21,8 @@ usage: aip-filter [-p|--print] <expr> [file...] * Input is newline-delimited JSON, with one value per line. Matching lines are preserved. * An empty filter copies the selected input unchanged; `--print` with an empty filter prints nothing. * Invalid JSON lines are reported and skipped. Any invalid JSON or input/output error causes a nonzero exit status. +* Records are limited to 8 MiB by default, including the newline. `--max-record-bytes N` sets a positive byte limit. Oversized records are reported and skipped without buffering the rest of the line; later records are still processed and the exit status indicates failure. The limit does not apply to an empty filter, which copies bytes directly. +* Memory depends on the largest allowed record and its parsed JSON values, not on the total number of records. The record limit is not a process-memory limit. ### Matching rules diff --git a/src/main.rs b/src/main.rs index afeaf39..c93b169 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,46 +2,51 @@ use aip_filter::{parse, CompiledFilter}; use serde_json::Value as Json; use std::{ fs::File, - io::{self, BufRead, BufReader, BufWriter, Write}, + io::{self, BufRead, BufReader, BufWriter, Read, Write}, path::Path, process, }; +const DEFAULT_MAX_RECORD_BYTES: usize = 8 * 1024 * 1024; + fn filter_lines( expr: Option<&CompiledFilter<'_>>, mut reader: impl BufRead, out: &mut impl Write, source: &str, + max_record_bytes: usize, ) -> io::Result<bool> { let Some(expr) = expr else { io::copy(&mut reader, out)?; return Ok(true); }; - let mut line_buffer = String::with_capacity(128); + let mut line_buffer = Vec::with_capacity(128); let mut lineno = 0; let mut valid = true; - while reader.read_line(&mut line_buffer)? > 0 { + while (&mut reader).take(max_record_bytes as u64 + 1) + .read_until(b'\n', &mut line_buffer)? > 0 + { lineno += 1; - - // Skip empty lines - if !line_buffer.trim().is_empty() { - match serde_json::from_str::<Json>(&line_buffer) { + if line_buffer.len() > max_record_bytes { + eprintln!("{source}:{lineno}: record exceeds {max_record_bytes} bytes"); + valid = false; + if line_buffer.last() != Some(&b'\n') { + reader.skip_until(b'\n')?; + } + } else if !line_buffer.iter().all(u8::is_ascii_whitespace) { + match serde_json::from_slice::<Json>(&line_buffer) { Ok(json) => { if expr.evaluate(&json) { - // Write directly to stdout - out.write_all(line_buffer.as_bytes())?; + out.write_all(&line_buffer)?; } } Err(e) => { - eprintln!("{}:{}: invalid JSON: {}", source, lineno, e); + eprintln!("{source}:{lineno}: invalid JSON: {e}"); valid = false; } } } - - // Clear the buffer for the next line, - // which keeps the memory allocated but resets the length to 0. line_buffer.clear(); } @@ -49,27 +54,31 @@ fn filter_lines( } fn usage(program: &str) -> ! { - eprintln!("usage: {program} [-p|--print] <expr> [file...]"); + eprintln!("usage: {program} [-p|--print] [--max-record-bytes N] [--] <expr> [file...]"); process::exit(2); } fn main() { - let mut args: Vec<String> = std::env::args().collect(); - let program = args.remove(0); - - if args.is_empty() || args[0] == "-h" || args[0] == "--help" { - usage(&program); - } - - // Check for print flag + let mut args = std::env::args(); + let program = args.next().unwrap(); let mut print_only = false; - if args[0] == "-p" || args[0] == "--print" { - print_only = true; - args.remove(0); - if args.is_empty() { usage(&program); } - } - - let raw_expr = args.remove(0); + let mut max_record_bytes = DEFAULT_MAX_RECORD_BYTES; + let raw_expr = loop { + let arg = args.next().unwrap_or_else(|| usage(&program)); + match arg.as_str() { + "-p" | "--print" => print_only = true, + "--max-record-bytes" => { + max_record_bytes = args.next() + .and_then(|value| value.parse::<usize>().ok()) + .filter(|value| *value > 0 && value.checked_add(1).is_some()) + .unwrap_or_else(|| usage(&program)); + } + "--" => break args.next().unwrap_or_else(|| usage(&program)), + "-h" | "--help" => usage(&program), + _ => break arg, + } + }; + let mut paths = args.peekable(); let expr = match parse(&raw_expr) { Ok(expr) => expr, Err(e) => { @@ -88,14 +97,14 @@ fn main() { } let filter = expr.as_ref().map(|expr| expr.compile()).transpose() .map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?; - if args.is_empty() { - return filter_lines(filter.as_ref(), io::stdin().lock(), &mut out, "<stdin>"); + if paths.peek().is_none() { + return filter_lines(filter.as_ref(), io::stdin().lock(), &mut out, "<stdin>", max_record_bytes); } let mut valid = true; - for path in &args { - let file = File::open(Path::new(path)) + for path in paths { + let file = File::open(Path::new(&path)) .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?; - valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut out, path) + valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut out, &path, max_record_bytes) .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?; } Ok(valid) diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index e86a29a..118f139 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -2,6 +2,10 @@ use std::io::Write; use std::process::{Command, Output, Stdio}; fn run(expr: &str, input: &str, args: &[&str]) -> Output { + run_bytes(expr, input.as_bytes(), args) +} + +fn run_bytes(expr: &str, input: &[u8], args: &[&str]) -> Output { let mut child = Command::new(env!("CARGO_BIN_EXE_aip-filter")) .args(args) .arg(expr) @@ -10,12 +14,7 @@ fn run(expr: &str, input: &str, args: &[&str]) -> Output { .stderr(Stdio::piped()) .spawn() .unwrap(); - child - .stdin - .take() - .unwrap() - .write_all(input.as_bytes()) - .unwrap(); + child.stdin.take().unwrap().write_all(input).unwrap(); child.wait_with_output().unwrap() } @@ -163,6 +162,83 @@ fn invalid_json_is_reported_and_returns_failure() { ); } +#[test] +fn record_limits_preserve_boundaries_and_continue_after_errors() { + let record = "\"match\"\n"; + let limit = record.len().to_string(); + let args = ["--max-record-bytes", limit.as_str()]; + let output = run("match", &(record.repeat(2)), &args); + assert!(output.status.success()); + assert_eq!(output.stdout, record.repeat(2).as_bytes()); + + for oversized in ["\"matchx\"\n", "\"a much longer match\"\n"] { + let output = run("match", &format!("{oversized}{record}"), &args); + assert_eq!(output.status.code(), Some(1)); + assert_eq!(output.stdout, record.as_bytes()); + assert!(String::from_utf8_lossy(&output.stderr).contains("<stdin>:1: record exceeds")); + } + let output = run("match", "\"an unterminated match", &args); + assert_eq!(output.status.code(), Some(1)); + assert!(output.stdout.is_empty()); + + let output = run("match", "\"match\"", &["--max-record-bytes", "7"]); + assert!(output.status.success()); + assert_eq!(output.stdout, b"\"match\""); + let output = run("match", "\"match\"\r\n", &["--max-record-bytes", "9"]); + assert!(output.status.success()); + assert_eq!(output.stdout, b"\"match\"\r\n"); +} + +#[test] +fn default_record_limit_is_enforced() { + let input = "x".repeat(8 * 1024 * 1024) + "\n\"match\"\n"; + let output = run("match", &input, &[]); + assert_eq!(output.status.code(), Some(1)); + assert_eq!(output.stdout, b"\"match\"\n"); + assert!(String::from_utf8_lossy(&output.stderr).contains("record exceeds 8388608 bytes")); +} + +#[test] +fn invalid_record_limit_arguments_fail() { + for limit in ["0", "-1", "invalid", "18446744073709551615"] { + let output = run("match", "", &["--max-record-bytes", limit]); + assert_eq!(output.status.code(), Some(2)); + } +} + +#[test] +fn invalid_utf8_is_reported_without_losing_later_records() { + let output = run_bytes("match", b"\"\xff\"\n\"match\"\n", &[]); + assert_eq!(output.status.code(), Some(1)); + assert_eq!(output.stdout, b"\"match\"\n"); + assert!(String::from_utf8_lossy(&output.stderr).contains("<stdin>:1: invalid JSON")); + let output = run_bytes("", b"\xff\n", &["--max-record-bytes", "1"]); + assert!(output.status.success()); + assert_eq!(output.stdout, b"\xff\n"); +} + +#[test] +fn filtering_multiple_files_preserves_successful_output() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join(format!("cli-test-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let first = dir.join("first.jsonl"); + let second = dir.join("second.jsonl"); + std::fs::write(&first, b"\"match first\"\n").unwrap(); + std::fs::write(&second, b"invalid\n\"match second\"\n").unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_aip-filter")) + .arg("match") + .arg(&first) + .arg(&second) + .output() + .unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + assert_eq!(output.status.code(), Some(1)); + assert_eq!(output.stdout, b"\"match first\"\n\"match second\"\n"); + assert!(String::from_utf8_lossy(&output.stderr).contains("second.jsonl:1: invalid JSON")); +} + #[cfg(unix)] #[test] fn output_flush_errors_are_reported() { |
