aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Cargo.lock154
-rw-r--r--Cargo.toml5
-rw-r--r--README.md13
-rw-r--r--src/eval.rs323
-rw-r--r--src/main.rs95
-rw-r--r--tests/cli_tests.rs187
-rw-r--r--tests/eval_tests.rs94
7 files changed, 472 insertions, 399 deletions
diff --git a/Cargo.lock b/Cargo.lock
index e0f1949..806a0d0 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -15,51 +15,8 @@ dependencies = [
name = "aip_filter"
version = "0.1.0"
dependencies = [
- "rand",
"regex",
- "serde",
"serde_json",
- "serde_yaml",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
-
-[[package]]
-name = "equivalent"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
-
-[[package]]
-name = "getrandom"
-version = "0.3.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
-dependencies = [
- "cfg-if",
- "libc",
- "r-efi",
- "wasip2",
-]
-
-[[package]]
-name = "hashbrown"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
-
-[[package]]
-name = "indexmap"
-version = "2.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff"
-dependencies = [
- "equivalent",
- "hashbrown",
]
[[package]]
@@ -69,27 +26,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
-name = "libc"
-version = "0.2.184"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
-
-[[package]]
name = "memchr"
version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
[[package]]
-name = "ppv-lite86"
-version = "0.2.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
-dependencies = [
- "zerocopy",
-]
-
-[[package]]
name = "proc-macro2"
version = "1.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -108,41 +50,6 @@ dependencies = [
]
[[package]]
-name = "r-efi"
-version = "5.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
-
-[[package]]
-name = "rand"
-version = "0.9.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
-dependencies = [
- "rand_chacha",
- "rand_core",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.9.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
name = "regex"
version = "1.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -172,19 +79,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
-name = "ryu"
-version = "1.0.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
-
-[[package]]
name = "serde"
version = "1.0.228"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
dependencies = [
"serde_core",
- "serde_derive",
]
[[package]]
@@ -221,19 +121,6 @@ dependencies = [
]
[[package]]
-name = "serde_yaml"
-version = "0.9.34+deprecated"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
-dependencies = [
- "indexmap",
- "itoa",
- "ryu",
- "serde",
- "unsafe-libyaml",
-]
-
-[[package]]
name = "syn"
version = "2.0.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -251,47 +138,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
-name = "unsafe-libyaml"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
-
-[[package]]
-name = "wasip2"
-version = "1.0.2+wasi-0.2.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
-dependencies = [
- "wit-bindgen",
-]
-
-[[package]]
-name = "wit-bindgen"
-version = "0.51.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5"
-
-[[package]]
-name = "zerocopy"
-version = "0.8.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
-dependencies = [
- "zerocopy-derive",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.8.48"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
-]
-
-[[package]]
name = "zmij"
version = "1.0.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 521ca86..6e2d5e4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,11 +6,6 @@ edition = "2024"
[dependencies]
regex = "1"
serde_json = "1"
-rand = "0.9"
-
-[dev-dependencies]
-serde = { version = "1", features = ["derive"] }
-serde_yaml = "0.9"
[profile.release]
opt-level = "z"
diff --git a/README.md b/README.md
index c372d84..6d842e4 100644
--- a/README.md
+++ b/README.md
@@ -18,6 +18,17 @@ usage: aip-filter [-p|--print] <expr> [file...]
* `--print` prints the parsed expression instead of evaluating it.
* If no files are given, reads from stdin.
+* Input is newline-delimited JSON, with one value per line. Matching lines are preserved.
+* An empty filter copies the selected input unchanged; `--print` with an empty filter prints nothing.
+* Invalid JSON lines are reported and skipped. Any invalid JSON or input/output error causes a nonzero exit status.
+
+### Matching rules
+
+* Bare literals search values recursively, including nested objects and arrays, but not object keys. String searches are case-insensitive substrings.
+* An unquoted boolean field name evaluates that field. A dotted name resolves a field path first and otherwise becomes a global literal. Quote a search term to prevent field lookup, for example `'"active"'` or `'"example.com"'`.
+* String `=` and `!=` ignore ASCII case and support one leading or trailing `*`. The two operators are opposites.
+* String `:` matches words or phrases bounded by whitespace or ASCII punctuation, or a leading/trailing wildcard. Lists match any element; maps match keys. `field:*` tests for a non-default value, including a nonempty list or map.
+* Functions are parsed but not evaluated. Regexes are validated when parsing and recompiled during evaluation. Filter nesting is not bounded; do not accept untrusted filter expressions without external limits.
### Examples
@@ -41,6 +52,8 @@ echo '{"user": {"settings": {"theme": "dark"}}}' | \
Implement `Filterable` for your type, then call `Expr::evaluate`:
+Bare-literal matching also requires overriding `all_field_values` or `matches_global`; the default searches no values. Override `matches_global` to search borrowed data without building a list of values. `Value::matches_global` provides recursive matching for individual values.
+
```rust
use aip_filter::{parse, Value, Filterable};
diff --git a/src/eval.rs b/src/eval.rs
index bf2afec..24866b9 100644
--- a/src/eval.rs
+++ b/src/eval.rs
@@ -1,6 +1,6 @@
use super::ast::{BinaryOp, CompareOp, Expr, UnaryOp};
use regex::Regex;
-use std::cmp::Ordering;
+use std::{borrow::Cow, cmp::Ordering};
/// A runtime value used during filter evaluation.
#[derive(Debug, Clone)]
@@ -35,6 +35,11 @@ impl<'a> Value<'a> {
}
}
+ /// Matches a global literal against this value and any nested values.
+ pub fn matches_global(&self, search: &str) -> bool {
+ value_contains_str(self, search)
+ }
+
/// Returns true if this value is the zero/default for its type.
pub fn is_zero(&self) -> bool {
match self {
@@ -68,10 +73,14 @@ impl<'a> Value<'a> {
Some(a.to_lowercase().cmp(&b.to_lowercase()))
}
// Cross-numeric comparisons
- (Value::Int(a), Value::Float(b)) => (*a as f64).partial_cmp(b),
- (Value::Float(a), Value::Int(b)) => a.partial_cmp(&(*b as f64)),
- (Value::Uint(a), Value::Float(b)) => (*a as f64).partial_cmp(b),
- (Value::Float(a), Value::Uint(b)) => a.partial_cmp(&(*b as f64)),
+ (Value::Int(a), Value::Float(b)) => compare_integer_float(i128::from(*a), *b),
+ (Value::Float(a), Value::Int(b)) => {
+ compare_integer_float(i128::from(*b), *a).map(Ordering::reverse)
+ }
+ (Value::Uint(a), Value::Float(b)) => compare_integer_float(i128::from(*a), *b),
+ (Value::Float(a), Value::Uint(b)) => {
+ compare_integer_float(i128::from(*b), *a).map(Ordering::reverse)
+ }
(Value::Int(a), Value::Uint(b)) => {
if *a < 0 {
Some(Ordering::Less)
@@ -91,6 +100,23 @@ impl<'a> Value<'a> {
}
}
+// The integer is an i64 or u64. Comparing its exact value avoids f64 rounding above 2^53.
+fn compare_integer_float(integer: i128, float: f64) -> Option<Ordering> {
+ if float.is_nan() {
+ return None;
+ }
+ if float < i64::MIN as f64 {
+ return Some(Ordering::Greater);
+ }
+ if float >= u64::MAX as f64 {
+ return Some(Ordering::Less);
+ }
+ match integer.cmp(&(float as i128)) {
+ Ordering::Equal => 0.0_f64.partial_cmp(&float.fract()),
+ order => Some(order),
+ }
+}
+
/// Trait that types must implement to be filterable.
///
/// # Example
@@ -144,6 +170,13 @@ pub trait Filterable {
fn all_field_values(&self) -> Vec<Value<'_>> {
Vec::new()
}
+
+ /// Matches a bare literal across field values. Override to search without collecting values.
+ fn matches_global(&self, search: &str) -> bool {
+ self.all_field_values()
+ .iter()
+ .any(|value| value.matches_global(search))
+ }
}
impl Expr {
@@ -171,25 +204,10 @@ impl Expr {
Expr::Name {
name, is_string, ..
} => {
- // A bare name/value: global restriction matching.
- // Check if any field contains or equals the value.
- if let Some(field_val) = val.field(name) {
- // If it's a boolean field referenced by name, return its value
- if !*is_string
- && let Value::Bool(b) = field_val {
- return b;
- }
- return !field_val.is_zero();
- }
-
- // Try matching against all fields
- let search = name.as_str();
- for field_val in val.all_field_values() {
- if value_contains_str(&field_val, search) {
- return true;
- }
+ if !*is_string && let Some(Value::Bool(b)) = val.field(name) {
+ return b;
}
- false
+ val.matches_global(name)
}
Expr::Member {
@@ -202,7 +220,7 @@ impl Expr {
_ => !val.is_zero(),
}
} else {
- false
+ expr_to_literal_str(self).is_some_and(|literal| val.matches_global(&literal))
}
}
@@ -228,14 +246,13 @@ fn eval_comparison<T: Filterable>(
return false;
};
- eval_comparison_with_value(op, &left_val, right, val)
+ eval_comparison_with_value(op, &left_val, right)
}
-fn eval_comparison_with_value<T: Filterable>(
+fn eval_comparison_with_value(
op: CompareOp,
left_val: &Value<'_>,
right: &Expr,
- val: &T,
) -> bool {
// Handle the right side, it could be a composite expression (AND/OR)
match right {
@@ -244,23 +261,28 @@ fn eval_comparison_with_value<T: Filterable>(
left: rleft,
right: rright,
..
- } => {
- let l = eval_comparison_with_value(op, left_val, rleft, val);
- let r = eval_comparison_with_value(op, left_val, rright, val);
- match bin_op {
- BinaryOp::And => l && r,
- BinaryOp::Or => l || r,
+ } => match bin_op {
+ BinaryOp::And => {
+ eval_comparison_with_value(op, left_val, rleft)
+ && eval_comparison_with_value(op, left_val, rright)
}
- }
+ BinaryOp::Or => {
+ eval_comparison_with_value(op, left_val, rleft)
+ || eval_comparison_with_value(op, left_val, rright)
+ }
+ },
Expr::Unary {
op: UnaryOp::Minus | UnaryOp::Not,
expr,
..
- } => !eval_comparison_with_value(op, left_val, expr, val),
+ } => !eval_comparison_with_value(op, left_val, expr),
_ => {
let right_str = expr_to_literal_str(right);
let right_str = right_str.as_deref();
+ if op == CompareOp::Has && right_str == Some("*") {
+ return !left_val.is_zero();
+ }
match (&left_val, op) {
// Has operator on lists
@@ -300,10 +322,6 @@ fn eval_comparison_with_value<T: Filterable>(
// Has on strings; substring/word boundary match
(_, CompareOp::Has) => {
if let Some(rs) = right_str {
- // Wildcard handling
- if rs == "*" {
- return !left_val.is_zero();
- }
value_matches_str(left_val, rs, op)
} else {
false
@@ -344,14 +362,14 @@ fn eval_comparison_with_value<T: Filterable>(
}
/// Resolve an expression to its value given a filterable object.
-fn resolve_expr_value<'a, T: Filterable>(expr: &Expr, val: &'a T) -> Option<Value<'a>> {
+fn resolve_expr_value<'a, T: Filterable>(expr: &'a Expr, val: &'a T) -> Option<Value<'a>> {
match expr {
Expr::Name {
name, is_string, ..
} => {
if *is_string {
// A quoted string on the left side of a comparison is just a string literal
- Some(Value::StringOwned(name.clone()))
+ Some(Value::String(name))
} else {
val.field(name)
}
@@ -420,9 +438,9 @@ fn resolve_member_value<'a, T: Filterable>(
}
/// Extract the literal string from an expression (name or member chain as dotted string).
-fn expr_to_literal_str(expr: &Expr) -> Option<String> {
+fn expr_to_literal_str(expr: &Expr) -> Option<Cow<'_, str>> {
match expr {
- Expr::Name { name, .. } => Some(name.clone()),
+ Expr::Name { name, .. } => Some(Cow::Borrowed(name)),
Expr::Member { holder, member, .. } => {
// a.b on the right side is just "a.b" as a string
let mut parts = vec![member.as_str()];
@@ -441,7 +459,7 @@ fn expr_to_literal_str(expr: &Expr) -> Option<String> {
}
}
parts.reverse();
- Some(parts.join("."))
+ Some(Cow::Owned(parts.join(".")))
}
Expr::Function { .. } => None,
_ => None,
@@ -451,29 +469,21 @@ fn expr_to_literal_str(expr: &Expr) -> Option<String> {
/// Parse a literal string to match the type of a reference value.
fn parse_literal_for_value<'a>(reference: &Value<'_>, literal: &'a str) -> Value<'a> {
match reference {
- Value::Bool(_) => match literal.to_lowercase().as_str() {
- "true" | "1" => Value::Bool(true),
- "false" | "0" => Value::Bool(false),
- _ => Value::String(literal),
- },
- Value::Int(_) => {
- if let Ok(i) = literal.parse::<i64>() {
- Value::Int(i)
- } else if let Ok(f) = literal.parse::<f64>() {
- Value::Int(f as i64)
+ Value::Bool(_) => {
+ if literal.eq_ignore_ascii_case("true") || literal == "1" {
+ Value::Bool(true)
+ } else if literal.eq_ignore_ascii_case("false") || literal == "0" {
+ Value::Bool(false)
} else {
Value::String(literal)
}
}
- Value::Uint(_) => {
- if let Ok(u) = literal.parse::<u64>() {
+ Value::Int(_) | Value::Uint(_) | Value::Float(_) => {
+ if let Ok(i) = literal.parse::<i64>() {
+ Value::Int(i)
+ } else if let Ok(u) = literal.parse::<u64>() {
Value::Uint(u)
- } else {
- Value::String(literal)
- }
- }
- Value::Float(_) => {
- if let Ok(f) = literal.parse::<f64>() {
+ } else if let Ok(f) = literal.parse::<f64>() {
Value::Float(f)
} else {
Value::String(literal)
@@ -489,42 +499,8 @@ fn value_matches_str(val: &Value<'_>, s: &str, op: CompareOp) -> bool {
match val {
Value::String(vs) => string_match(vs, s, op),
Value::StringOwned(vs) => string_match(vs.as_str(), s, op),
- Value::Int(i) => {
- if let Ok(ri) = s.parse::<i64>() {
- match op {
- CompareOp::Equals | CompareOp::Has => *i == ri,
- _ => false,
- }
- } else {
- false
- }
- }
- Value::Uint(u) => {
- if let Ok(ru) = s.parse::<u64>() {
- match op {
- CompareOp::Equals | CompareOp::Has => *u == ru,
- _ => false,
- }
- } else {
- false
- }
- }
- Value::Float(f) => {
- if let Ok(rf) = s.parse::<f64>() {
- match op {
- CompareOp::Equals | CompareOp::Has => (*f - rf).abs() < f64::EPSILON,
- _ => false,
- }
- } else {
- false
- }
- }
- Value::Bool(b) => {
- let sb = matches!(s.to_lowercase().as_str(), "true" | "1");
- match op {
- CompareOp::Equals | CompareOp::Has => *b == sb,
- _ => false,
- }
+ Value::Int(_) | Value::Uint(_) | Value::Float(_) | Value::Bool(_) => {
+ compare_values(op, val, &parse_literal_for_value(val, s))
}
Value::List(items) => items.iter().any(|item| value_matches_str(item, s, op)),
Value::Map(entries) => entries.iter().any(|(_, v)| value_matches_str(v, s, op)),
@@ -532,87 +508,75 @@ fn value_matches_str(val: &Value<'_>, s: &str, op: CompareOp) -> bool {
}
}
-/// String matching: case-insensitive equality for `=`, word-boundary match for `:`, wildcard support.
+/// Equality and `:` support a leading or trailing wildcard.
+/// Word boundaries for `:` are whitespace or ASCII punctuation.
fn string_match(haystack: &str, needle: &str, op: CompareOp) -> bool {
+ if op == CompareOp::NotEquals {
+ return !string_match(haystack, needle, CompareOp::Equals);
+ }
+ if matches!(op, CompareOp::Equals | CompareOp::Has) {
+ if let Some(suffix) = needle.strip_prefix('*') {
+ return haystack.len()
+ .checked_sub(suffix.len())
+ .and_then(|start| haystack.get(start..))
+ .is_some_and(|tail| tail.eq_ignore_ascii_case(suffix));
+ }
+ if let Some(prefix) = needle.strip_suffix('*') {
+ return haystack.get(..prefix.len())
+ .is_some_and(|head| head.eq_ignore_ascii_case(prefix));
+ }
+ }
match op {
+ CompareOp::Equals => haystack.eq_ignore_ascii_case(needle),
CompareOp::Has => {
- // Wildcard: * prefix
- if let Some(suffix) = needle.strip_prefix('*') {
- return haystack.len() >= suffix.len()
- && haystack[haystack.len() - suffix.len()..].eq_ignore_ascii_case(suffix);
- }
- // Wildcard: * suffix
- if let Some(prefix) = needle.strip_suffix('*') {
- return haystack.len() >= prefix.len()
- && haystack[..prefix.len()].eq_ignore_ascii_case(prefix);
+ let haystack = lowercase(haystack);
+ let needle = lowercase(needle);
+ if needle.is_empty() {
+ return true;
}
-
- // Word boundary match
- let hl = haystack.to_lowercase();
- let nl = needle.to_lowercase();
- if let Some(idx) = hl.find(&nl) {
- let is_boundary = |b: u8| -> bool {
- b < 0x80 && !b.is_ascii_alphanumeric()
- };
- let start_ok = idx == 0 || is_boundary(haystack.as_bytes()[idx - 1]);
- let end_ok = idx + nl.len() >= haystack.len()
- || is_boundary(haystack.as_bytes()[idx + nl.len()]);
- if start_ok && end_ok {
+ let boundary = |c: char| {
+ c.is_whitespace() || (c.is_ascii() && !c.is_ascii_alphanumeric())
+ };
+ let mut offset = 0;
+ while let Some(index) = haystack[offset..].find(needle.as_ref()) {
+ let start = offset + index;
+ let end = start + needle.len();
+ if haystack[..start].chars().next_back().is_none_or(boundary)
+ && haystack[end..].chars().next().is_none_or(boundary)
+ {
return true;
}
+ offset = start + haystack[start..].chars().next().unwrap().len_utf8();
}
-
- // Also check individual words
- let words = split_string(haystack);
- if words.len() > 1 {
- for word in &words {
- if string_match(word, needle, CompareOp::Has) {
- return true;
- }
- }
- }
-
false
}
- CompareOp::Equals => {
- // Wildcard: * prefix
- if let Some(suffix) = needle.strip_prefix('*') {
- return haystack.len() >= suffix.len()
- && haystack[haystack.len() - suffix.len()..].eq_ignore_ascii_case(suffix);
- }
- // Wildcard: * suffix
- if let Some(prefix) = needle.strip_suffix('*') {
- return haystack.len() >= prefix.len()
- && haystack[..prefix.len()].eq_ignore_ascii_case(prefix);
- }
- haystack.eq_ignore_ascii_case(needle)
- }
- CompareOp::NotEquals => !haystack.eq_ignore_ascii_case(needle),
- _ => {
- // For ordering comparisons on strings, use case-sensitive compare
- let ord = haystack.cmp(needle);
- match op {
- CompareOp::LessThan => ord == Ordering::Less,
- CompareOp::LessThanEquals => ord != Ordering::Greater,
- CompareOp::GreaterThan => ord == Ordering::Greater,
- CompareOp::GreaterThanEquals => ord != Ordering::Less,
- _ => false,
- }
- }
+ _ => compare_values(op, &Value::String(haystack), &Value::String(needle)),
}
}
+fn lowercase(s: &str) -> Cow<'_, str> {
+ if s.is_ascii() && !s.bytes().any(|b| b.is_ascii_uppercase()) {
+ Cow::Borrowed(s)
+ } else {
+ Cow::Owned(s.to_lowercase())
+ }
+}
+
+fn contains_ignore_case(haystack: &str, needle: &str) -> bool {
+ lowercase(haystack).contains(lowercase(needle).as_ref())
+}
+
/// Check if a value contains a string (for global restriction matching).
fn value_contains_str(val: &Value<'_>, search: &str) -> bool {
match val {
- Value::String(s) => s.to_lowercase().contains(&search.to_lowercase()),
- Value::StringOwned(s) => s.to_lowercase().contains(&search.to_lowercase()),
+ Value::String(s) => contains_ignore_case(s, search),
+ Value::StringOwned(s) => contains_ignore_case(s, search),
Value::Int(i) => i.to_string() == search,
Value::Uint(u) => u.to_string() == search,
Value::Float(f) => f.to_string() == search,
Value::Bool(b) => {
let bs = if *b { "true" } else { "false" };
- bs == search.to_lowercase()
+ bs.eq_ignore_ascii_case(search)
}
Value::List(items) => items.iter().any(|item| value_contains_str(item, search)),
Value::Map(entries) => entries.iter().any(|(_, v)| value_contains_str(v, search)),
@@ -631,7 +595,7 @@ fn compare_values(op: CompareOp, left: &Value<'_>, right: &Value<'_>) -> bool {
}
CompareOp::NotEquals => {
if let (Some(ls), Some(rs)) = (left.as_str(), right.as_str()) {
- return !ls.eq_ignore_ascii_case(rs);
+ return string_match(ls, rs, CompareOp::NotEquals);
}
left.compare(right) != Some(Ordering::Equal)
}
@@ -652,45 +616,6 @@ fn compare_values(op: CompareOp, left: &Value<'_>, right: &Value<'_>) -> bool {
}
}
-/// Split a string into words.
-fn split_string(s: &str) -> Vec<&str> {
- let mut result = Vec::new();
- let mut start = None;
-
- for (i, c) in s.char_indices() {
- if c.is_whitespace() {
- if let Some(s_idx) = start {
- result.push(&s[s_idx..i]);
- start = None;
- }
- } else if is_text_start_char(c) {
- if start.is_none() {
- start = Some(i);
- }
- } else {
- if let Some(s_idx) = start {
- result.push(&s[s_idx..i]);
- start = None;
- }
- // Single char becomes its own word
- let end = i + c.len_utf8();
- result.push(&s[i..end]);
- }
- }
-
- if let Some(s_idx) = start {
- result.push(&s[s_idx..]);
- }
-
- result
-}
-
-fn is_text_start_char(r: char) -> bool {
- matches!(r,
- '#'..='\'' | '*' | '/' | ';' | '?' | '@' | 'A'..='Z' | '[' | ']' | '^'..='}' | '\\'
- ) || ('\u{00a1}'..='\u{0effff}').contains(&r)
-}
-
#[cfg(test)]
mod tests {
use super::*;
diff --git a/src/main.rs b/src/main.rs
index 5bd138a..e8b58b0 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -25,7 +25,19 @@ impl<'j> Filterable for JsonRecord<'j> {
fn all_field_values(&self) -> Vec<Value<'_>> {
match self.0 {
Json::Object(map) => map.values().filter_map(json_to_value).collect(),
- _ => vec![],
+ _ => json_to_value(self.0).into_iter().collect(),
+ }
+ }
+
+ fn matches_global(&self, search: &str) -> bool {
+ match self.0 {
+ Json::Object(map) => map
+ .values()
+ .any(|value| JsonRecord(value).matches_global(search)),
+ Json::Array(items) => items
+ .iter()
+ .any(|value| JsonRecord(value).matches_global(search)),
+ _ => json_to_value(self.0).is_some_and(|value| value.matches_global(search)),
}
}
}
@@ -43,19 +55,29 @@ fn json_to_value(v: &Json) -> Option<Value<'_>> {
Json::Array(arr) => Some(Value::List(
arr.iter().filter_map(json_to_value).collect(),
)),
- // Nested objects are accessible via field_path; skip at top level.
- Json::Object(_) => None,
+ Json::Object(map) => Some(Value::Map(
+ map.iter()
+ .filter_map(|(key, value)| {
+ json_to_value(value).map(|value| (key.as_str(), value))
+ })
+ .collect(),
+ )),
}
}
fn filter_lines(
- expr: &Expr,
+ expr: Option<&Expr>,
mut reader: impl BufRead,
out: &mut impl Write,
source: &str,
-) -> io::Result<()> {
+) -> io::Result<bool> {
+ let Some(expr) = expr else {
+ io::copy(&mut reader, out)?;
+ return Ok(true);
+ };
let mut line_buffer = String::with_capacity(128);
let mut lineno = 0;
+ let mut valid = true;
while reader.read_line(&mut line_buffer)? > 0 {
lineno += 1;
@@ -71,6 +93,7 @@ fn filter_lines(
}
Err(e) => {
eprintln!("{}:{}: invalid JSON: {}", source, lineno, e);
+ valid = false;
}
}
}
@@ -80,7 +103,7 @@ fn filter_lines(
line_buffer.clear();
}
- Ok(())
+ Ok(valid)
}
fn usage(program: &str) -> ! {
@@ -106,50 +129,40 @@ fn main() {
let raw_expr = args.remove(0);
let expr = match parse(&raw_expr) {
- Ok(Some(e)) => e,
- Ok(None) => {
- // Empty expression matches everything; just cat the input.
- let stdin = io::stdin();
- let mut out = BufWriter::new(io::stdout().lock());
- for line in stdin.lock().lines() {
- writeln!(out, "{}", line.unwrap_or_default()).ok();
- }
- return;
- }
+ Ok(expr) => expr,
Err(e) => {
eprintln!("{program}: {e}");
process::exit(1);
}
};
- if print_only {
- println!("{}", expr);
- return;
- }
-
let mut out = BufWriter::new(io::stdout().lock());
-
- if args.is_empty() {
- // Read from stdin.
- let stdin = io::stdin();
- if let Err(e) = filter_lines(&expr, stdin.lock(), &mut out, "<stdin>") {
- eprintln!("{program}: {e}");
- process::exit(1);
+ let result = (|| -> io::Result<bool> {
+ if print_only {
+ if let Some(expr) = &expr {
+ writeln!(out, "{expr}")?;
+ }
+ return Ok(true);
}
- } else {
+ if args.is_empty() {
+ return filter_lines(expr.as_ref(), io::stdin().lock(), &mut out, "<stdin>");
+ }
+ let mut valid = true;
for path in &args {
- let source = path.as_str();
- let file = match File::open(Path::new(path)) {
- Ok(f) => f,
- Err(e) => {
- eprintln!("{program}: {path}: {e}");
- process::exit(1);
- }
- };
- if let Err(e) = filter_lines(&expr, BufReader::new(file), &mut out, source) {
- eprintln!("{program}: {path}: {e}");
- process::exit(1);
- }
+ let file = File::open(Path::new(path))
+ .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
+ valid &= filter_lines(expr.as_ref(), BufReader::new(file), &mut out, path)
+ .map_err(|e| io::Error::new(e.kind(), format!("{path}: {e}")))?;
+ }
+ Ok(valid)
+ })();
+ let flushed = out.flush();
+ match result.and_then(|valid| flushed.map(|()| valid)) {
+ Ok(true) => {}
+ Ok(false) => process::exit(1),
+ Err(e) => {
+ eprintln!("{program}: {e}");
+ process::exit(1);
}
}
}
diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs
new file mode 100644
index 0000000..e86a29a
--- /dev/null
+++ b/tests/cli_tests.rs
@@ -0,0 +1,187 @@
+use std::io::Write;
+use std::process::{Command, Output, Stdio};
+
+fn run(expr: &str, input: &str, args: &[&str]) -> Output {
+ let mut child = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
+ .args(args)
+ .arg(expr)
+ .stdin(Stdio::piped())
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ child
+ .stdin
+ .take()
+ .unwrap()
+ .write_all(input.as_bytes())
+ .unwrap();
+ child.wait_with_output().unwrap()
+}
+
+fn matches(expr: &str, record: &str, expected: bool) {
+ let input = format!("{record}\n");
+ let output = run(expr, &input, &[]);
+ assert!(
+ output.status.success(),
+ "{expr}: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ assert_eq!(
+ String::from_utf8(output.stdout).unwrap(),
+ if expected { input } else { String::new() },
+ "{expr} against {record}"
+ );
+}
+
+#[test]
+fn bare_literals_search_nested_values() {
+ for record in [
+ r#"{"msg":"hello foobar"}"#,
+ r#"{"nested":{"msg":"hello foobar"}}"#,
+ r#"{"events":[{"msg":"hello foobar"}]}"#,
+ r#"{"events":[["hello foobar"]]}"#,
+ r#"["hello foobar"]"#,
+ r#""hello foobar""#,
+ ] {
+ matches("foobar", record, true);
+ matches("FOOBAR", record, true);
+ matches("absent", record, false);
+ }
+ matches("foobar", r#"{"foobar":"unrelated"}"#, false);
+ matches("foobar", r#"{"foobar":"","msg":"hello foobar"}"#, true);
+ matches("foo.bar", r#"{"msg":"hello foo.bar"}"#, true);
+ matches("active", r#"{"active":true}"#, true);
+ matches("active", r#"{"active":false}"#, false);
+}
+
+#[test]
+fn quoted_literals_do_not_resolve_fields() {
+ matches(r#""msg""#, r#"{"msg":"unrelated"}"#, false);
+ matches(r#""active""#, r#"{"active":false,"msg":"active"}"#, true);
+}
+
+#[test]
+fn unicode_wildcards_do_not_panic() {
+ for expr in [
+ r#"name = "x*""#,
+ r#"name = "*x""#,
+ r#"name : "x*""#,
+ r#"name : "*x""#,
+ ] {
+ matches(expr, r#"{"name":"é"}"#, false);
+ }
+ matches(r#"name = "é*""#, r#"{"name":"étude"}"#, true);
+ matches(r#"name != "op*""#, r#"{"name":"open"}"#, false);
+ matches(r#"name != "op*""#, r#"{"name":"closed"}"#, true);
+ matches(r#"names != "op*""#, r#"{"names":["open"]}"#, false);
+}
+
+#[test]
+fn has_checks_later_phrases_and_unicode_boundaries() {
+ matches(
+ r#"msg:"hello world""#,
+ r#"{"msg":"xhello world; hello world"}"#,
+ true,
+ );
+ matches(r#"msg:"a a""#, r#"{"msg":"xa a a"}"#, true);
+ matches("msg:a", r#"{"msg":"İa"}"#, false);
+ matches("msg:a", r#"{"msg":"İ a"}"#, true);
+ matches("msg:crash", r#"{"msg":"crashes"}"#, false);
+}
+
+#[test]
+fn numeric_comparisons_do_not_truncate_or_round_integers() {
+ for (expr, record, expected) in [
+ ("n = 5.9", r#"{"n":5}"#, false),
+ ("n < 5.9", r#"{"n":5}"#, true),
+ ("n > -5.9", r#"{"n":-5}"#, true),
+ ("n = -5", r#"{"n":-5}"#, true),
+ ("n = 9007199254740992.0", r#"{"n":9007199254740993}"#, false),
+ ("n = 9007199254740993", r#"{"n":9007199254740992.0}"#, false),
+ ("n > -1", r#"{"n":18446744073709551615}"#, true),
+ (
+ "n < 18446744073709551616.0",
+ r#"{"n":18446744073709551615}"#,
+ true,
+ ),
+ (
+ "n < -9223372036854775808.0",
+ r#"{"n":-9223372036854775808}"#,
+ false,
+ ),
+ ("n:0.0", r#"{"n":[0.00000000000000001]}"#, false),
+ ("n:5.0", r#"{"n":[5]}"#, true),
+ ] {
+ matches(expr, record, expected);
+ }
+}
+
+#[test]
+fn has_and_boolean_values() {
+ matches("flags:garbage", r#"{"flags":[false]}"#, false);
+ matches("flags:false", r#"{"flags":[false]}"#, true);
+ matches("flags:FALSE", r#"{"flags":[false]}"#, true);
+ matches("flags:0", r#"{"flags":[false]}"#, true);
+ matches("tags:*", r#"{"tags":[0]}"#, true);
+ matches("tags:*", r#"{"tags":[]}"#, false);
+ matches("meta:owner", r#"{"meta":{"owner":"alice"}}"#, true);
+ matches("meta:*", r#"{"meta":{"owner":"alice"}}"#, true);
+ matches("meta:*", r#"{"meta":{}}"#, false);
+ matches("meta.owner=alice", r#"{"meta":{"owner":"alice"}}"#, true);
+}
+
+#[test]
+fn empty_filter_and_print_mode() {
+ let input = "{\"msg\":\"foobar\"}\n";
+ let output = run("", input, &[]);
+ assert!(output.status.success());
+ assert_eq!(output.stdout, input.as_bytes());
+ let output = run("", input, &["--print"]);
+ assert!(output.status.success());
+ assert!(output.stdout.is_empty());
+
+ let path = format!("{}/README.md", env!("CARGO_MANIFEST_DIR"));
+ let output = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
+ .args(["", &path])
+ .stdin(Stdio::null())
+ .output()
+ .unwrap();
+ assert!(output.status.success());
+ assert_eq!(output.stdout, std::fs::read(path).unwrap());
+}
+
+#[test]
+fn invalid_json_is_reported_and_returns_failure() {
+ let output = run("foobar", "not json\n{\"msg\":\"foobar\"}\n", &[]);
+ assert!(!output.status.success());
+ assert_eq!(output.stdout, b"{\"msg\":\"foobar\"}\n");
+ assert!(
+ String::from_utf8(output.stderr)
+ .unwrap()
+ .contains("<stdin>:1: invalid JSON")
+ );
+}
+
+#[cfg(unix)]
+#[test]
+fn output_flush_errors_are_reported() {
+ use std::os::fd::OwnedFd;
+ use std::os::unix::net::UnixStream;
+ let (writer, reader) = UnixStream::pair().unwrap();
+ drop(reader);
+ let mut child = Command::new(env!("CARGO_BIN_EXE_aip-filter"))
+ .arg("foobar")
+ .stdin(Stdio::piped())
+ .stdout(Stdio::from(OwnedFd::from(writer)))
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ child
+ .stdin
+ .take()
+ .unwrap()
+ .write_all(b"{\"msg\":\"foobar\"}\n")
+ .unwrap();
+ assert!(!child.wait_with_output().unwrap().status.success());
+}
diff --git a/tests/eval_tests.rs b/tests/eval_tests.rs
new file mode 100644
index 0000000..885b497
--- /dev/null
+++ b/tests/eval_tests.rs
@@ -0,0 +1,94 @@
+use aip_filter::{Filterable, Value, parse};
+
+struct Record<'a> {
+ value: Value<'a>,
+}
+
+impl Filterable for Record<'_> {
+ fn field(&self, name: &str) -> Option<Value<'_>> {
+ (name == "value").then(|| self.value.clone())
+ }
+
+ fn all_field_values(&self) -> Vec<Value<'_>> {
+ vec![self.value.clone()]
+ }
+}
+
+fn matches(value: Value<'_>, filter: &str) -> bool {
+ parse(filter).unwrap().unwrap().evaluate(&Record { value })
+}
+
+#[test]
+fn global_matching_for_library_records() {
+ let value = Value::Map(vec![
+ ("ignored_key", Value::Null),
+ (
+ "events",
+ Value::List(vec![Value::StringOwned("FOObar".into())]),
+ ),
+ ]);
+ assert!(matches(value.clone(), "foobar"));
+ assert!(!matches(value.clone(), "ignored_key"));
+ assert!(value.matches_global("foobar"));
+ assert!(Value::String("ÉTUDE").matches_global("étude"));
+ assert!(Value::String("").matches_global(""));
+ assert!(!Value::Null.matches_global("null"));
+}
+
+#[test]
+fn global_matching_can_avoid_collecting_values() {
+ struct Direct;
+ impl Filterable for Direct {
+ fn field(&self, _: &str) -> Option<Value<'_>> {
+ None
+ }
+
+ fn all_field_values(&self) -> Vec<Value<'_>> {
+ panic!("global matching must use the override");
+ }
+
+ fn matches_global(&self, search: &str) -> bool {
+ Value::String("hello foobar").matches_global(search)
+ }
+ }
+ assert!(parse("foobar").unwrap().unwrap().evaluate(&Direct));
+}
+
+#[test]
+fn integer_float_boundaries() {
+ for (integer, float, order) in [
+ (Value::Int(i64::MIN), i64::MIN as f64, "="),
+ (Value::Int(i64::MAX), i64::MAX as f64, "<"),
+ (Value::Uint(u64::MAX), u64::MAX as f64, "<"),
+ (Value::Int(0), -0.0, "="),
+ (Value::Uint(0), -0.5, ">"),
+ (Value::Int(-5), -5.9, ">"),
+ (Value::Int(5), 5.9, "<"),
+ (Value::Int(1), f64::INFINITY, "<"),
+ (Value::Uint(1), f64::NEG_INFINITY, ">"),
+ ] {
+ assert!(matches(integer, &format!("value {order} \"{float:?}\"")));
+ }
+ assert!(!matches(Value::Int(0), "value = NaN"));
+ assert!(!matches(Value::Float(f64::NAN), "value > 0"));
+ assert!(matches(Value::Float(f64::INFINITY), "value = inf"));
+ assert!(matches(Value::Float(f64::NEG_INFINITY), "value < -1"));
+}
+
+#[test]
+fn composite_comparisons_and_negation() {
+ for (filter, expected) in [
+ ("value:(open OR closed)", true),
+ ("value:(open AND closed)", false),
+ ("value:(NOT closed)", true),
+ (r#"value =~ ("^op" OR "^cl")"#, true),
+ (r#"value !~ ("^cl" AND "^sh")"#, true),
+ (r#"value != "op*""#, false),
+ ] {
+ assert_eq!(
+ matches(Value::StringOwned("open".into()), filter),
+ expected,
+ "{filter}"
+ );
+ }
+}