diff options
| -rw-r--r-- | Cargo.lock | 154 | ||||
| -rw-r--r-- | Cargo.toml | 5 | ||||
| -rw-r--r-- | README.md | 13 | ||||
| -rw-r--r-- | src/eval.rs | 323 | ||||
| -rw-r--r-- | src/main.rs | 95 | ||||
| -rw-r--r-- | tests/cli_tests.rs | 187 | ||||
| -rw-r--r-- | tests/eval_tests.rs | 94 |
7 files changed, 472 insertions, 399 deletions
@@ -15,51 +15,8 @@ dependencies = [ name = "aip_filter" version = "0.1.0" dependencies = [ - "rand", "regex", - "serde", "serde_json", - "serde_yaml", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasip2", -] - -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - -[[package]] -name = "indexmap" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" -dependencies = [ - "equivalent", - "hashbrown", ] [[package]] @@ -69,27 +26,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] -name = "libc" -version = "0.2.184" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" - -[[package]] name = "memchr" version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] name = "proc-macro2" version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -108,41 +50,6 @@ dependencies = [ ] [[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom", -] - -[[package]] name = "regex" version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -172,19 +79,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - -[[package]] name = "serde" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", - "serde_derive", ] [[package]] @@ -221,19 +121,6 @@ dependencies = [ ] [[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - -[[package]] name = "syn" version = "2.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -251,47 +138,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" [[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "wasip2" -version = "1.0.2+wasi-0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" - -[[package]] -name = "zerocopy" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.48" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6,11 +6,6 @@ edition = "2024" [dependencies] regex = "1" serde_json = "1" -rand = "0.9" - -[dev-dependencies] -serde = { version = "1", features = ["derive"] } -serde_yaml = "0.9" [profile.release] opt-level = "z" @@ -18,6 +18,17 @@ usage: aip-filter [-p|--print] <expr> [file...] * `--print` prints the parsed expression instead of evaluating it. * If no files are given, reads from stdin. +* Input is newline-delimited JSON, with one value per line. Matching lines are preserved. +* An empty filter copies the selected input unchanged; `--print` with an empty filter prints nothing. +* Invalid JSON lines are reported and skipped. Any invalid JSON or input/output error causes a nonzero exit status. + +### Matching rules + +* Bare literals search values recursively, including nested objects and arrays, but not object keys. String searches are case-insensitive substrings. +* An unquoted boolean field name evaluates that field. A dotted name resolves a field path first and otherwise becomes a global literal. Quote a search term to prevent field lookup, for example `'"active"'` or `'"example.com"'`. +* String `=` and `!=` ignore ASCII case and support one leading or trailing `*`. The two operators are opposites. +* String `:` matches words or phrases bounded by whitespace or ASCII punctuation, or a leading/trailing wildcard. Lists match any element; maps match keys. `field:*` tests for a non-default value, including a nonempty list or map. +* Functions are parsed but not evaluated. Regexes are validated when parsing and recompiled during evaluation. Filter nesting is not bounded; do not accept untrusted filter expressions without external limits. ### Examples @@ -41,6 +52,8 @@ echo '{"user": {"settings": {"theme": "dark"}}}' | \ Implement `Filterable` for your type, then call `Expr::evaluate`: +Bare-literal matching also requires overriding `all_field_values` or `matches_global`; the default searches no values. Override `matches_global` to search borrowed data without building a list of values. `Value::matches_global` provides recursive matching for individual values. + ```rust use aip_filter::{parse, Value, Filterable}; diff --git a/src/eval.rs b/src/eval.rs index bf2afec..24866b9 100644 --- a/src/eval.rs +++ b/src/eval.rs @@ -1,6 +1,6 @@ use super::ast::{BinaryOp, CompareOp, Expr, UnaryOp}; use regex::Regex; -use std::cmp::Ordering; +use std::{borrow::Cow, cmp::Ordering}; /// A runtime value used during filter evaluation. #[derive(Debug, Clone)] @@ -35,6 +35,11 @@ impl<'a> Value<'a> { } } + /// Matches a global literal against this value and any nested values. + pub fn matches_global(&self, search: &str) -> bool { + value_contains_str(self, search) + } + /// Returns true if this value is the zero/default for its type. pub fn is_zero(&self) -> bool { match self { @@ -68,10 +73,14 @@ impl<'a> Value<'a> { Some(a.to_lowercase().cmp(&b.to_lowercase())) } // Cross-numeric comparisons - (Value::Int(a), Value::Float(b)) => (*a as f64).partial_cmp(b), - (Value::Float(a), Value::Int(b)) => a.partial_cmp(&(*b as f64)), - (Value::Uint(a), Value::Float(b)) => (*a as f64).partial_cmp(b), - (Value::Float(a), Value::Uint(b)) => a.partial_cmp(&(*b as f64)), + (Value::Int(a), Value::Float(b)) => compare_integer_float(i128::from(*a), *b), + (Value::Float(a), Value::Int(b)) => { + compare_integer_float(i128::from(*b), *a).map(Ordering::reverse) + } + (Value::Uint(a), Value::Float(b)) => compare_integer_float(i128::from(*a), *b), + (Value::Float(a), Value::Uint(b)) => { + compare_integer_float(i128::from(*b), *a).map(Ordering::reverse) + } (Value::Int(a), Value::Uint(b)) => { if *a < 0 { Some(Ordering::Less) @@ -91,6 +100,23 @@ impl<'a> Value<'a> { } } +// The integer is an i64 or u64. Comparing its exact value avoids f64 rounding above 2^53. +fn compare_integer_float(integer: i128, float: f64) -> Option<Ordering> { + if float.is_nan() { + return None; + } + if float < i64::MIN as f64 { + return Some(Ordering::Greater); + } + if float >= u64::MAX as f64 { + return Some(Ordering::Less); + } + match integer.cmp(&(float as i128)) { + Ordering::Equal => 0.0_f64.partial_cmp(&float.fract()), + order => Some(order), + } +} + /// Trait that types must implement to be filterable. /// /// # Example @@ -144,6 +170,13 @@ pub trait Filterable { fn all_field_values(&self) -> Vec<Value<'_>> { Vec::new() } + + /// Matches a bare literal across field values. Override to search without collecting values. + fn matches_global(&self, search: &str) -> bool { + self.all_field_values() + .iter() + .any(|value| value.matches_global(search)) + } } impl Expr { @@ -171,25 +204,10 @@ impl Expr { Expr::Name { name, is_string, .. } => { - // A bare name/value: global restriction matching. - // Check if any field contains or equals the value. - if let Some(field_val) = val.field(name) { - // If it's a boolean field referenced by name, return its value - if !*is_string - && let Value::Bool(b) = field_val { - return b; - } - return !field_val.is_zero(); - } - - // Try matching against all fields - let search = name.as_str(); - for field_val in val.all_field_values() { - if value_contains_str(&field_val, search) { - return true; - } + if !*is_string && let Some(Value::Bool(b)) = val.field(name) { + return b; } - false + val.matches_global(name) } Expr::Member { @@ -202,7 +220,7 @@ impl Expr { _ => !val.is_zero(), } } else { - false + expr_to_literal_str(self).is_some_and(|literal| val.matches_global(&literal)) } } @@ -228,14 +246,13 @@ fn eval_comparison<T: Filterable>( return false; }; - eval_comparison_with_value(op, &left_val, right, val) + eval_comparison_with_value(op, &left_val, right) } -fn eval_comparison_with_value<T: Filterable>( +fn eval_comparison_with_value( op: CompareOp, left_val: &Value<'_>, right: &Expr, - val: &T, ) -> bool { // Handle the right side, it could be a composite expression (AND/OR) match right { @@ -244,23 +261,28 @@ fn eval_comparison_with_value<T: Filterable>( left: rleft, right: rright, .. - } => { - let l = eval_comparison_with_value(op, left_val, rleft, val); - let r = eval_comparison_with_value(op, left_val, rright, val); - match bin_op { - BinaryOp::And => l && r, - BinaryOp::Or => l || r, + } => match bin_op { + BinaryOp::And => { + eval_comparison_with_value(op, left_val, rleft) + && eval_comparison_with_value(op, left_val, rright) } - } + BinaryOp::Or => { + eval_comparison_with_value(op, left_val, rleft) + || eval_comparison_with_value(op, left_val, rright) + } + }, Expr::Unary { op: UnaryOp::Minus | UnaryOp::Not, expr, .. - } => !eval_comparison_with_value(op, left_val, expr, val), + } => !eval_comparison_with_value(op, left_val, expr), _ => { let right_str = expr_to_literal_str(right); let right_str = right_str.as_deref(); + if op == CompareOp::Has && right_str == Some("*") { + return !left_val.is_zero(); + } match (&left_val, op) { // Has operator on lists @@ -300,10 +322,6 @@ fn eval_comparison_with_value<T: Filterable>( // Has on strings; substring/word boundary match (_, CompareOp::Has) => { if let Some(rs) = right_str { - // Wildcard handling - if rs == "*" { - return !left_val.is_zero(); - } value_matches_str(left_val, rs, op) } else { false @@ -344,14 +362,14 @@ fn eval_comparison_with_value<T: Filterable>( } /// Resolve an expression to its value given a filterable object. -fn resolve_expr_value<'a, T: Filterable>(expr: &Expr, val: &'a T) -> Option<Value<'a>> { +fn resolve_expr_value<'a, T: Filterable>(expr: &'a Expr, val: &'a T) -> Option<Value<'a>> { match expr { Expr::Name { name, is_string, .. } => { if *is_string { // A quoted string on the left side of a comparison is just a string literal - Some(Value::StringOwned(name.clone())) + Some(Value::String(name)) } else { val.field(name) } @@ -420,9 +438,9 @@ fn resolve_member_value<'a, T: Filterable>( } /// Extract the literal string from an expression (name or member chain as dotted string). -fn expr_to_literal_str(expr: &Expr) -> Option<String> { +fn expr_to_literal_str(expr: &Expr) -> Option<Cow<'_, str>> { match expr { - Expr::Name { name, .. } => Some(name.clone()), + Expr::Name { name, .. } => Some(Cow::Borrowed(name)), Expr::Member { holder, member, .. } => { // a.b on the right side is just "a.b" as a string let mut parts = vec![member.as_str()]; @@ -441,7 +459,7 @@ fn expr_to_literal_str(expr: &Expr) -> Option<String> { } } parts.reverse(); - Some(parts.join(".")) + Some(Cow::Owned(parts.join("."))) } Expr::Function { .. } => None, _ => None, @@ -451,29 +469,21 @@ fn expr_to_literal_str(expr: &Expr) -> Option<String> { /// Parse a literal string to match the type of a reference value. fn parse_literal_for_value<'a>(reference: &Value<'_>, literal: &'a str) -> Value<'a> { match reference { - Value::Bool(_) => match literal.to_lowercase().as_str() { - "true" | "1" => Value::Bool(true), - "false" | "0" => Value::Bool(false), - _ => Value::String(literal), - }, - Value::Int(_) => { - if let Ok(i) = literal.parse::<i64>() { - Value::Int(i) - } else if let Ok(f) = literal.parse::<f64>() { - Value::Int(f as i64) + Value::Bool(_) => { + if literal.eq_ignore_ascii_case("true") || literal == "1" { + Value::Bool(true) + } else if literal.eq_ignore_ascii_case("false") || literal == "0" { + Value::Bool(false) } else { Value::String(literal) } } - Value::Uint(_) => { - if let Ok(u) = literal.parse::<u64>() { + Value::Int(_) | Value::Uint(_) | Value::Float(_) => { + if let Ok(i) = literal.parse::<i64>() { + Value::Int(i) + } else if let Ok(u) = literal.parse::<u64>() { Value::Uint(u) - } else { - Value::String(literal) - } - } - Value::Float(_) => { - if let Ok(f) = literal.parse::<f64>() { + } else if let Ok(f) = literal.parse::<f64>() { Value::Float(f) } else { Value::String(literal) @@ -489,42 +499,8 @@ fn value_matches_str(val: &Value<'_>, s: &str, op: CompareOp) -> bool { match val { Value::String(vs) => string_match(vs, s, op), Value::StringOwned(vs) => string_match(vs.as_str(), s, op), - Value::Int(i) => { - if let Ok(ri) = s.parse::<i64>() { - match op { - CompareOp::Equals | CompareOp::Has => *i == ri, - _ => false, - } - } else { - false - } - } - Value::Uint(u) => { - if let Ok(ru) = s.parse::<u64>() { - match op { - CompareOp::Equals | CompareOp::Has => *u == ru, - _ => false, - } - } else { - false - } - } - Value::Float(f) => { - if let Ok(rf) = s.parse::<f64>() { - match op { - CompareOp::Equals | CompareOp::Has => (*f - rf).abs() < f64::EPSILON, - _ => false, - } - } else { - false - } - } - Value::Bool(b) => { - let sb = matches!(s.to_lowercase().as_str(), "true" | "1"); - match op { - CompareOp::Equals | CompareOp::Has => *b == sb, - _ => false, - } + Value::Int(_) | Value::Uint(_) | Value::Float(_) | Value::Bool(_) => { + compare_values(op, val, &parse_literal_for_value(val, s)) } Value::List(items) => items.iter().any(|item| value_matches_str(item, s, op)), Value::Map(entries) => entries.iter().any(|(_, v)| value_matches_str(v, s, op)), @@ -532,87 +508,75 @@ fn value_matches_str(val: &Value<'_>, s: &str, op: CompareOp) -> bool { } } -/// String matching: case-insensitive equality for `=`, word-boundary match for `:`, wildcard support. +/// Equality and `:` support a leading or trailing wildcard. +/// Word boundaries for `:` are whitespace or ASCII punctuation. fn string_match(haystack: &str, needle: &str, op: CompareOp) -> bool { + if op == CompareOp::NotEquals { + return !string_match(haystack, needle, CompareOp::Equals); + } + if matches!(op, CompareOp::Equals | CompareOp::Has) { + if let Some(suffix) = needle.strip_prefix('*') { + return haystack.len() + .checked_sub(suffix.len()) + .and_then(|start| haystack.get(start..)) + .is_some_and(|tail| tail.eq_ignore_ascii_case(suffix)); + } + if let Some(prefix) = needle.strip_suffix('*') { + return haystack.get(..prefix.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(prefix)); + } + } match op { + CompareOp::Equals => haystack.eq_ignore_ascii_case(needle), CompareOp::Has => { - // Wildcard: * prefix - if let Some(suffix) = needle.strip_prefix('*') { - return haystack.len() >= suffix.len() - && haystack[haystack.len() - suffix.len()..].eq_ignore_ascii_case(suffix); - } - // Wildcard: * suffix - if let Some(prefix) = needle.strip_suffix('*') { - return haystack.len() >= prefix.len() - && haystack[..prefix.len()].eq_ignore_ascii_case(prefix); + let haystack = lowercase(haystack); + let needle = lowercase(needle); + if needle.is_empty() { + return true; } - - // Word boundary match - let hl = haystack.to_lowercase(); - let nl = needle.to_lowercase(); - if let Some(idx) = hl.find(&nl) { - let is_boundary = |b: u8| -> bool { - b < 0x80 && !b.is_ascii_alphanumeric() - }; - let start_ok = idx == 0 || is_boundary(haystack.as_bytes()[idx - 1]); - let end_ok = idx + nl.len() >= haystack.len() - || is_boundary(haystack.as_bytes()[idx + nl.len()]); - if start_ok && end_ok { + let boundary = |c: char| { + c.is_whitespace() || (c.is_ascii() && !c.is_ascii_alphanumeric()) + }; + let mut offset = 0; + while let Some(index) = haystack[offset..].find(needle.as_ref()) { + let start = offset + index; + let end = start + needle.len(); + if haystack[..start].chars().next_back().is_none_or(boundary) + && haystack[end..].chars().next().is_none_or(boundary) + { return true; } + offset = start + haystack[start..].chars().next().unwrap().len_utf8(); } - - // Also check individual words - let words = split_string(haystack); - if words.len() > 1 { - for word in &words { - if string_match(word, needle, CompareOp::Has) { - return true; - } - } - } - false } - CompareOp::Equals => { - // Wildcard: * prefix - if let Some(suffix) = needle.strip_prefix('*') { - return haystack.len() >= suffix.len() - && haystack[haystack.len() - suffix.len()..].eq_ignore_ascii_case(suffix); - } - // Wildcard: * suffix - if let Some(prefix) = needle.strip_suffix('*') { - return haystack.len() >= prefix.len() - && haystack[..prefix.len()].eq_ignore_ascii_case(prefix); - } - haystack.eq_ignore_ascii_case(needle) - } - CompareOp::NotEquals => !haystack.eq_ignore_ascii_case(needle), - _ => { - // For ordering comparisons on strings, use case-sensitive compare - let ord = haystack.cmp(needle); - match op { - CompareOp::LessThan => ord == Ordering::Less, - CompareOp::LessThanEquals => ord != Ordering::Greater, - CompareOp::GreaterThan => ord == Ordering::Greater, - CompareOp::GreaterThanEquals => ord != Ordering::Less, - _ => false, - } - } + _ => compare_values(op, &Value::String(haystack), &Value::String(needle)), } } +fn lowercase(s: &str) -> Cow<'_, str> { + if s.is_ascii() && !s.bytes().any(|b| b.is_ascii_uppercase()) { + Cow::Borrowed(s) + } else { + Cow::Owned(s.to_lowercase()) + } +} + +fn contains_ignore_case(haystack: &str, needle: &str) -> bool { + lowercase(haystack).contains(lowercase(needle).as_ref()) +} + /// Check if a value contains a string (for global restriction matching). fn value_contains_str(val: &Value<'_>, search: &str) -> bool { match val { - Value::String(s) => s.to_lowercase().contains(&search.to_lowercase()), - Value::StringOwned(s) => s.to_lowercase().contains(&search.to_lowercase()), + Value::String(s) => contains_ignore_case(s, search), + Value::StringOwned(s) => contains_ignore_case(s, search), Value::Int(i) => i.to_string() == search, Value::Uint(u) => u.to_string() == search, Value::Float(f) => f.to_string() == search, Value::Bool(b) => { let bs = if *b { "true" } else { "false" }; - bs == search.to_lowercase() + bs.eq_ignore_ascii_case(search) } Value::List(items) => items.iter().any(|item| value_contains_str(item, search)), Value::Map(entries) => entries.iter().any(|(_, v)| value_contains_str(v, search)), @@ -631,7 +595,7 @@ fn compare_values(op: CompareOp, left: &Value<'_>, right: &Value<'_>) -> bool { } CompareOp::NotEquals => { if let (Some(ls), Some(rs)) = (left.as_str(), right.as_str()) { - return !ls.eq_ignore_ascii_case(rs); + return string_match(ls, rs, CompareOp::NotEquals); } left.compare(right) != Some(Ordering::Equal) } @@ -652,45 +616,6 @@ fn compare_values(op: CompareOp, left: &Value<'_>, right: &Value<'_>) -> bool { } } -/// Split a string into words. -fn split_string(s: &str) -> Vec<&str> { - let mut result = Vec::new(); - let mut start = None; - - for (i, c) in s.char_indices() { - if c.is_whitespace() { - if let Some(s_idx) = start { - result.push(&s[s_idx..i]); - start = None; - } - } else if is_text_start_char(c) { - if start.is_none() { - start = Some(i); - } - } else { - if let Some(s_idx) = start { - result.push(&s[s_idx..i]); - start = None; - } - // Single char becomes its own word - let end = i + c.len_utf8(); - result.push(&s[i..end]); - } - } - - if let Some(s_idx) = start { - result.push(&s[s_idx..]); - } - - result -} - -fn is_text_start_char(r: char) -> bool { - matches!(r, - '#'..='\'' | '*' | '/' | ';' | '?' | '@' | 'A'..='Z' | '[' | ']' | '^'..='}' | '\\' - ) || ('\u{00a1}'..='\u{0effff}').contains(&r) -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/main.rs b/src/main.rs index 5bd138a..e8b58b0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,19 @@ impl<'j> Filterable for JsonRecord<'j> { fn all_field_values(&self) -> Vec<Value<'_>> { match self.0 { Json::Object(map) => map.values().filter_map(json_to_value).collect(), - _ => vec![], + _ => json_to_value(self.0).into_iter().collect(), + } + } + + fn matches_global(&self, search: &str) -> bool { + match self.0 { + Json::Object(map) => map + .values() + .any(|value| JsonRecord(value).matches_global(search)), + Json::Array(items) => items + .iter() + .any(|value| JsonRecord(value).matches_global(search)), + _ => json_to_value(self.0).is_some_and(|value| value.matches_global(search)), } } } @@ -43,19 +55,29 @@ fn json_to_value(v: &Json) -> Option<Value<'_>> { Json::Array(arr) => Some(Value::List( arr.iter().filter_map(json_to_value).collect(), )), - // Nested objects are accessible via field_path; skip at top level. - Json::Object(_) => None, + Json::Object(map) => Some(Value::Map( + map.iter() + .filter_map(|(key, value)| { + json_to_value(value).map(|value| (key.as_str(), value)) + }) + .collect(), + )), } } fn filter_lines( - expr: &Expr, + expr: Option<&Expr>, mut reader: impl BufRead, out: &mut impl Write, source: &str, -) -> io::Result<()> { +) -> io::Result<bool> { + let Some(expr) = expr else { + io::copy(&mut reader, out)?; + return Ok(true); + }; let mut line_buffer = String::with_capacity(128); let mut lineno = 0; + let mut valid = true; while reader.read_line(&mut line_buffer)? > 0 { lineno += 1; @@ -71,6 +93,7 @@ fn filter_lines( } Err(e) => { eprintln!("{}:{}: invalid JSON: {}", source, lineno, e); + valid = false; } } } @@ -80,7 +103,7 @@ fn filter_lines( line_buffer.clear(); } - Ok(()) + Ok(valid) } fn usage(program: &str) -> ! { @@ -106,50 +129,40 @@ fn main() { let raw_expr = args.remove(0); let expr = match parse(&raw_expr) { - Ok(Some(e)) => e, - Ok(None) => { - // Empty expression matches everything; just cat the input. - let stdin = io::stdin(); - let mut out = BufWriter::new(io::stdout().lock()); - for line in stdin.lock().lines() { - writeln!(out, "{}", line.unwrap_or_default()).ok(); - } - return; - } + Ok(expr) => expr, Err(e) => { eprintln!("{program}: {e}"); process::exit(1); } }; - if print_only { - println!("{}", expr); - return; - } - let mut out = BufWriter::new(io::stdout().lock()); - - if args.is_empty() { - // Read from stdin. - let stdin = io::stdin(); - if let Err(e) = filter_lines(&expr, stdin.lock(), &mut out, "<stdin>") { - eprintln!("{program}: {e}"); - process::exit(1); + let result = (|| -> io::Result<bool> { + if print_only { + if let Some(expr) = &expr { + writeln!(out, "{expr}")?; + } + return Ok(true); } - } else { + if args.is_empty() { + return filter_lines(expr.as_ref(), io::stdin().lock(), &mut out, "<stdin>"); + } + let mut valid = true; for path in &args { - let source = path.as_str(); - let file = match File::open(Path::new(path)) { - Ok(f) => f, - Err(e) => { - eprintln!("{program}: {path}: {e}"); - process::exit(1); - } - }; - if let Err(e) = filter_lines(&expr, BufReader::new(file), &mut out, source) { - eprintln!("{program}: {path}: {e}"); - process::exit(1); - } + let file = File::open(Path::new(path)) + .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?; + valid &= filter_lines(expr.as_ref(), BufReader::new(file), &mut out, path) + .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?; + } + Ok(valid) + })(); + let flushed = out.flush(); + match result.and_then(|valid| flushed.map(|()| valid)) { + Ok(true) => {} + Ok(false) => process::exit(1), + Err(e) => { + eprintln!("{program}: {e}"); + process::exit(1); } } } diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs new file mode 100644 index 0000000..e86a29a --- /dev/null +++ b/tests/cli_tests.rs @@ -0,0 +1,187 @@ +use std::io::Write; +use std::process::{Command, Output, Stdio}; + +fn run(expr: &str, input: &str, args: &[&str]) -> Output { + let mut child = Command::new(env!("CARGO_BIN_EXE_aip-filter")) + .args(args) + .arg(expr) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(input.as_bytes()) + .unwrap(); + child.wait_with_output().unwrap() +} + +fn matches(expr: &str, record: &str, expected: bool) { + let input = format!("{record}\n"); + let output = run(expr, &input, &[]); + assert!( + output.status.success(), + "{expr}: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8(output.stdout).unwrap(), + if expected { input } else { String::new() }, + "{expr} against {record}" + ); +} + +#[test] +fn bare_literals_search_nested_values() { + for record in [ + r#"{"msg":"hello foobar"}"#, + r#"{"nested":{"msg":"hello foobar"}}"#, + r#"{"events":[{"msg":"hello foobar"}]}"#, + r#"{"events":[["hello foobar"]]}"#, + r#"["hello foobar"]"#, + r#""hello foobar""#, + ] { + matches("foobar", record, true); + matches("FOOBAR", record, true); + matches("absent", record, false); + } + matches("foobar", r#"{"foobar":"unrelated"}"#, false); + matches("foobar", r#"{"foobar":"","msg":"hello foobar"}"#, true); + matches("foo.bar", r#"{"msg":"hello foo.bar"}"#, true); + matches("active", r#"{"active":true}"#, true); + matches("active", r#"{"active":false}"#, false); +} + +#[test] +fn quoted_literals_do_not_resolve_fields() { + matches(r#""msg""#, r#"{"msg":"unrelated"}"#, false); + matches(r#""active""#, r#"{"active":false,"msg":"active"}"#, true); +} + +#[test] +fn unicode_wildcards_do_not_panic() { + for expr in [ + r#"name = "x*""#, + r#"name = "*x""#, + r#"name : "x*""#, + r#"name : "*x""#, + ] { + matches(expr, r#"{"name":"é"}"#, false); + } + matches(r#"name = "é*""#, r#"{"name":"étude"}"#, true); + matches(r#"name != "op*""#, r#"{"name":"open"}"#, false); + matches(r#"name != "op*""#, r#"{"name":"closed"}"#, true); + matches(r#"names != "op*""#, r#"{"names":["open"]}"#, false); +} + +#[test] +fn has_checks_later_phrases_and_unicode_boundaries() { + matches( + r#"msg:"hello world""#, + r#"{"msg":"xhello world; hello world"}"#, + true, + ); + matches(r#"msg:"a a""#, r#"{"msg":"xa a a"}"#, true); + matches("msg:a", r#"{"msg":"İa"}"#, false); + matches("msg:a", r#"{"msg":"İ a"}"#, true); + matches("msg:crash", r#"{"msg":"crashes"}"#, false); +} + +#[test] +fn numeric_comparisons_do_not_truncate_or_round_integers() { + for (expr, record, expected) in [ + ("n = 5.9", r#"{"n":5}"#, false), + ("n < 5.9", r#"{"n":5}"#, true), + ("n > -5.9", r#"{"n":-5}"#, true), + ("n = -5", r#"{"n":-5}"#, true), + ("n = 9007199254740992.0", r#"{"n":9007199254740993}"#, false), + ("n = 9007199254740993", r#"{"n":9007199254740992.0}"#, false), + ("n > -1", r#"{"n":18446744073709551615}"#, true), + ( + "n < 18446744073709551616.0", + r#"{"n":18446744073709551615}"#, + true, + ), + ( + "n < -9223372036854775808.0", + r#"{"n":-9223372036854775808}"#, + false, + ), + ("n:0.0", r#"{"n":[0.00000000000000001]}"#, false), + ("n:5.0", r#"{"n":[5]}"#, true), + ] { + matches(expr, record, expected); + } +} + +#[test] +fn has_and_boolean_values() { + matches("flags:garbage", r#"{"flags":[false]}"#, false); + matches("flags:false", r#"{"flags":[false]}"#, true); + matches("flags:FALSE", r#"{"flags":[false]}"#, true); + matches("flags:0", r#"{"flags":[false]}"#, true); + matches("tags:*", r#"{"tags":[0]}"#, true); + matches("tags:*", r#"{"tags":[]}"#, false); + matches("meta:owner", r#"{"meta":{"owner":"alice"}}"#, true); + matches("meta:*", r#"{"meta":{"owner":"alice"}}"#, true); + matches("meta:*", r#"{"meta":{}}"#, false); + matches("meta.owner=alice", r#"{"meta":{"owner":"alice"}}"#, true); +} + +#[test] +fn empty_filter_and_print_mode() { + let input = "{\"msg\":\"foobar\"}\n"; + let output = run("", input, &[]); + assert!(output.status.success()); + assert_eq!(output.stdout, input.as_bytes()); + let output = run("", input, &["--print"]); + assert!(output.status.success()); + assert!(output.stdout.is_empty()); + + let path = format!("{}/README.md", env!("CARGO_MANIFEST_DIR")); + let output = Command::new(env!("CARGO_BIN_EXE_aip-filter")) + .args(["", &path]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, std::fs::read(path).unwrap()); +} + +#[test] +fn invalid_json_is_reported_and_returns_failure() { + let output = run("foobar", "not json\n{\"msg\":\"foobar\"}\n", &[]); + assert!(!output.status.success()); + assert_eq!(output.stdout, b"{\"msg\":\"foobar\"}\n"); + assert!( + String::from_utf8(output.stderr) + .unwrap() + .contains("<stdin>:1: invalid JSON") + ); +} + +#[cfg(unix)] +#[test] +fn output_flush_errors_are_reported() { + use std::os::fd::OwnedFd; + use std::os::unix::net::UnixStream; + let (writer, reader) = UnixStream::pair().unwrap(); + drop(reader); + let mut child = Command::new(env!("CARGO_BIN_EXE_aip-filter")) + .arg("foobar") + .stdin(Stdio::piped()) + .stdout(Stdio::from(OwnedFd::from(writer))) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .take() + .unwrap() + .write_all(b"{\"msg\":\"foobar\"}\n") + .unwrap(); + assert!(!child.wait_with_output().unwrap().status.success()); +} diff --git a/tests/eval_tests.rs b/tests/eval_tests.rs new file mode 100644 index 0000000..885b497 --- /dev/null +++ b/tests/eval_tests.rs @@ -0,0 +1,94 @@ +use aip_filter::{Filterable, Value, parse}; + +struct Record<'a> { + value: Value<'a>, +} + +impl Filterable for Record<'_> { + fn field(&self, name: &str) -> Option<Value<'_>> { + (name == "value").then(|| self.value.clone()) + } + + fn all_field_values(&self) -> Vec<Value<'_>> { + vec![self.value.clone()] + } +} + +fn matches(value: Value<'_>, filter: &str) -> bool { + parse(filter).unwrap().unwrap().evaluate(&Record { value }) +} + +#[test] +fn global_matching_for_library_records() { + let value = Value::Map(vec![ + ("ignored_key", Value::Null), + ( + "events", + Value::List(vec![Value::StringOwned("FOObar".into())]), + ), + ]); + assert!(matches(value.clone(), "foobar")); + assert!(!matches(value.clone(), "ignored_key")); + assert!(value.matches_global("foobar")); + assert!(Value::String("ÉTUDE").matches_global("étude")); + assert!(Value::String("").matches_global("")); + assert!(!Value::Null.matches_global("null")); +} + +#[test] +fn global_matching_can_avoid_collecting_values() { + struct Direct; + impl Filterable for Direct { + fn field(&self, _: &str) -> Option<Value<'_>> { + None + } + + fn all_field_values(&self) -> Vec<Value<'_>> { + panic!("global matching must use the override"); + } + + fn matches_global(&self, search: &str) -> bool { + Value::String("hello foobar").matches_global(search) + } + } + assert!(parse("foobar").unwrap().unwrap().evaluate(&Direct)); +} + +#[test] +fn integer_float_boundaries() { + for (integer, float, order) in [ + (Value::Int(i64::MIN), i64::MIN as f64, "="), + (Value::Int(i64::MAX), i64::MAX as f64, "<"), + (Value::Uint(u64::MAX), u64::MAX as f64, "<"), + (Value::Int(0), -0.0, "="), + (Value::Uint(0), -0.5, ">"), + (Value::Int(-5), -5.9, ">"), + (Value::Int(5), 5.9, "<"), + (Value::Int(1), f64::INFINITY, "<"), + (Value::Uint(1), f64::NEG_INFINITY, ">"), + ] { + assert!(matches(integer, &format!("value {order} \"{float:?}\""))); + } + assert!(!matches(Value::Int(0), "value = NaN")); + assert!(!matches(Value::Float(f64::NAN), "value > 0")); + assert!(matches(Value::Float(f64::INFINITY), "value = inf")); + assert!(matches(Value::Float(f64::NEG_INFINITY), "value < -1")); +} + +#[test] +fn composite_comparisons_and_negation() { + for (filter, expected) in [ + ("value:(open OR closed)", true), + ("value:(open AND closed)", false), + ("value:(NOT closed)", true), + (r#"value =~ ("^op" OR "^cl")"#, true), + (r#"value !~ ("^cl" AND "^sh")"#, true), + (r#"value != "op*""#, false), + ] { + assert_eq!( + matches(Value::StringOwned("open".into()), filter), + expected, + "{filter}" + ); + } +} |
