aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--README.md4
-rw-r--r--src/main.rs77
-rw-r--r--tests/cli_tests.rs88
3 files changed, 128 insertions, 41 deletions
diff --git a/README.md b/README.md
index 4c1656b..056f76c 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@ cargo install --path .
## CLI usage
```bash
-usage: aip-filter [-p|--print] <expr> [file...]
+usage: aip-filter [-p|--print] [--max-record-bytes N] [--] <expr> [file...]
```
* `--print` prints the parsed expression instead of evaluating it.
@@ -21,6 +21,8 @@ usage: aip-filter [-p|--print] <expr> [file...]
* Input is newline-delimited JSON, with one value per line. Matching lines are preserved.
* An empty filter copies the selected input unchanged; `--print` with an empty filter prints nothing.
* Invalid JSON lines are reported and skipped. Any invalid JSON or input/output error causes a nonzero exit status.
+* Records are limited to 8 MiB by default, including the newline. `--max-record-bytes N` sets a positive byte limit. Oversized records are reported and skipped without buffering the rest of the line; later records are still processed and the exit status indicates failure. The limit does not apply to an empty filter, which copies bytes directly.
+* Memory depends on the largest allowed record and its parsed JSON values, not on the total number of records. The record limit is not a process-memory limit.
### Matching rules
diff --git a/src/main.rs b/src/main.rs
index afeaf39..c93b169 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -2,46 +2,51 @@ use aip_filter::{parse, CompiledFilter};
use serde_json::Value as Json;
use std::{
fs::File,
- io::{self, BufRead, BufReader, BufWriter, Write},
+ io::{self, BufRead, BufReader, BufWriter, Read, Write},
path::Path,
process,
};
+const DEFAULT_MAX_RECORD_BYTES: usize = 8 * 1024 * 1024;
+
fn filter_lines(
expr: Option<&CompiledFilter<'_>>,
mut reader: impl BufRead,
out: &mut impl Write,
source: &str,
+ max_record_bytes: usize,
) -> io::Result<bool> {
let Some(expr) = expr else {
io::copy(&mut reader, out)?;
return Ok(true);
};
- let mut line_buffer = String::with_capacity(128);
+ let mut line_buffer = Vec::with_capacity(128);
let mut lineno = 0;
let mut valid = true;
- while reader.read_line(&mut line_buffer)? > 0 {
+ while (&mut reader).take(max_record_bytes as u64 + 1)
+ .read_until(b'\n', &mut line_buffer)? > 0
+ {
lineno += 1;
-
- // Skip empty lines
- if !line_buffer.trim().is_empty() {
- match serde_json::from_str::<Json>(&line_buffer) {
+ if line_buffer.len() > max_record_bytes {
+ eprintln!("{source}:{lineno}: record exceeds {max_record_bytes} bytes");
+ valid = false;
+ if line_buffer.last() != Some(&b'\n') {
+ reader.skip_until(b'\n')?;
+ }
+ } else if !line_buffer.iter().all(u8::is_ascii_whitespace) {
+ match serde_json::from_slice::<Json>(&line_buffer) {
Ok(json) => {
if expr.evaluate(&json) {
- // Write directly to stdout
- out.write_all(line_buffer.as_bytes())?;
+ out.write_all(&line_buffer)?;
}
}
Err(e) => {
- eprintln!("{}:{}: invalid JSON: {}", source, lineno, e);
+ eprintln!("{source}:{lineno}: invalid JSON: {e}");
valid = false;
}
}
}
-
- // Clear the buffer for the next line,
- // which keeps the memory allocated but resets the length to 0.
line_buffer.clear();
}
@@ -49,27 +54,31 @@ fn filter_lines(
}
fn usage(program: &str) -> ! {
- eprintln!("usage: {program} [-p|--print] <expr> [file...]");
+ eprintln!("usage: {program} [-p|--print] [--max-record-bytes N] [--] <expr> [file...]");
process::exit(2);
}
fn main() {
- let mut args: Vec<String> = std::env::args().collect();
- let program = args.remove(0);
-
- if args.is_empty() || args[0] == "-h" || args[0] == "--help" {
- usage(&program);
- }
-
- // Check for print flag
+ let mut args = std::env::args();
+ let program = args.next().unwrap();
let mut print_only = false;
- if args[0] == "-p" || args[0] == "--print" {
- print_only = true;
- args.remove(0);
- if args.is_empty() { usage(&program); }
- }
-
- let raw_expr = args.remove(0);
+ let mut max_record_bytes = DEFAULT_MAX_RECORD_BYTES;
+ let raw_expr = loop {
+ let arg = args.next().unwrap_or_else(|| usage(&program));
+ match arg.as_str() {
+ "-p" | "--print" => print_only = true,
+ "--max-record-bytes" => {
+ max_record_bytes = args.next()
+ .and_then(|value| value.parse::<usize>().ok())
+ .filter(|value| *value > 0 && value.checked_add(1).is_some())
+ .unwrap_or_else(|| usage(&program));
+ }
+ "--" => break args.next().unwrap_or_else(|| usage(&program)),
+ "-h" | "--help" => usage(&program),
+ _ => break arg,
+ }
+ };
+ let mut paths = args.peekable();
let expr = match parse(&raw_expr) {
Ok(expr) => expr,
Err(e) => {
@@ -88,14 +97,14 @@ fn main() {
}
let filter = expr.as_ref().map(|expr| expr.compile()).transpose()
.map_err(|error| io::Error::new(io::ErrorKind::InvalidInput, error))?;
- if args.is_empty() {
- return filter_lines(filter.as_ref(), io::stdin().lock(), &mut out, "<stdin>");
+ if paths.peek().is_none() {
+ return filter_lines(filter.as_ref(), io::stdin().lock(), &mut out, "<stdin>", max_record_bytes);
}
let mut valid = true;
- for path in &args {
- let file = File::open(Path::new(path))
+ for path in paths {
+ let file = File::open(Path::new(&path))
.map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
- valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut out, path)
+ valid &= filter_lines(filter.as_ref(), BufReader::new(file), &mut out, &path, max_record_bytes)
.map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
}
Ok(valid)
diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs
index e86a29a..118f139 100644
--- a/tests/cli_tests.rs
+++ b/tests/cli_tests.rs
@@ -2,6 +2,10 @@ use std::io::Write;
use std::process::{Command, Output, Stdio};
fn run(expr: &str, input: &str, args: &[&str]) -> Output {
+ run_bytes(expr, input.as_bytes(), args)
+}
+
+fn run_bytes(expr: &str, input: &[u8], args: &[&str]) -> Output {
let mut child = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
.args(args)
.arg(expr)
@@ -10,12 +14,7 @@ fn run(expr: &str, input: &str, args: &[&str]) -> Output {
.stderr(Stdio::piped())
.spawn()
.unwrap();
- child
- .stdin
- .take()
- .unwrap()
- .write_all(input.as_bytes())
- .unwrap();
+ child.stdin.take().unwrap().write_all(input).unwrap();
child.wait_with_output().unwrap()
}
@@ -163,6 +162,83 @@ fn invalid_json_is_reported_and_returns_failure() {
);
}
+#[test]
+fn record_limits_preserve_boundaries_and_continue_after_errors() {
+ let record = "\"match\"\n";
+ let limit = record.len().to_string();
+ let args = ["--max-record-bytes", limit.as_str()];
+ let output = run("match", &(record.repeat(2)), &args);
+ assert!(output.status.success());
+ assert_eq!(output.stdout, record.repeat(2).as_bytes());
+
+ for oversized in ["\"matchx\"\n", "\"a much longer match\"\n"] {
+ let output = run("match", &format!("{oversized}{record}"), &args);
+ assert_eq!(output.status.code(), Some(1));
+ assert_eq!(output.stdout, record.as_bytes());
+ assert!(String::from_utf8_lossy(&output.stderr).contains("<stdin>:1: record exceeds"));
+ }
+ let output = run("match", "\"an unterminated match", &args);
+ assert_eq!(output.status.code(), Some(1));
+ assert!(output.stdout.is_empty());
+
+ let output = run("match", "\"match\"", &["--max-record-bytes", "7"]);
+ assert!(output.status.success());
+ assert_eq!(output.stdout, b"\"match\"");
+ let output = run("match", "\"match\"\r\n", &["--max-record-bytes", "9"]);
+ assert!(output.status.success());
+ assert_eq!(output.stdout, b"\"match\"\r\n");
+}
+
+#[test]
+fn default_record_limit_is_enforced() {
+ let input = "x".repeat(8 * 1024 * 1024) + "\n\"match\"\n";
+ let output = run("match", &input, &[]);
+ assert_eq!(output.status.code(), Some(1));
+ assert_eq!(output.stdout, b"\"match\"\n");
+ assert!(String::from_utf8_lossy(&output.stderr).contains("record exceeds 8388608 bytes"));
+}
+
+#[test]
+fn invalid_record_limit_arguments_fail() {
+ for limit in ["0", "-1", "invalid", "18446744073709551615"] {
+ let output = run("match", "", &["--max-record-bytes", limit]);
+ assert_eq!(output.status.code(), Some(2));
+ }
+}
+
+#[test]
+fn invalid_utf8_is_reported_without_losing_later_records() {
+ let output = run_bytes("match", b"\"\xff\"\n\"match\"\n", &[]);
+ assert_eq!(output.status.code(), Some(1));
+ assert_eq!(output.stdout, b"\"match\"\n");
+ assert!(String::from_utf8_lossy(&output.stderr).contains("<stdin>:1: invalid JSON"));
+ let output = run_bytes("", b"\xff\n", &["--max-record-bytes", "1"]);
+ assert!(output.status.success());
+ assert_eq!(output.stdout, b"\xff\n");
+}
+
+#[test]
+fn filtering_multiple_files_preserves_successful_output() {
+ let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("target")
+ .join(format!("cli-test-{}", std::process::id()));
+ std::fs::create_dir_all(&dir).unwrap();
+ let first = dir.join("first.jsonl");
+ let second = dir.join("second.jsonl");
+ std::fs::write(&first, b"\"match first\"\n").unwrap();
+ std::fs::write(&second, b"invalid\n\"match second\"\n").unwrap();
+ let output = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
+ .arg("match")
+ .arg(&first)
+ .arg(&second)
+ .output()
+ .unwrap();
+ std::fs::remove_dir_all(dir).unwrap();
+ assert_eq!(output.status.code(), Some(1));
+ assert_eq!(output.stdout, b"\"match first\"\n\"match second\"\n");
+ assert!(String::from_utf8_lossy(&output.stderr).contains("second.jsonl:1: invalid JSON"));
+}
+
#[cfg(unix)]
#[test]
fn output_flush_errors_are_reported() {