diff options
32 files changed, 4074 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..e0f1949 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,298 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "aip_filter" +version = "0.1.0" +dependencies = [ + "rand", + "regex", + "serde", + "serde_json", + "serde_yaml", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "indexmap" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zerocopy" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..521ca86 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "aip_filter" +version = "0.1.0" +edition = "2024" + +[dependencies] +regex = "1" +serde_json = "1" +rand = "0.9" + +[dev-dependencies] +serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" + +[profile.release] +opt-level = "z" +lto = true +codegen-units = 1 +panic = "abort" +strip = true + +[[bin]] +name = "aip-filter" +path = "src/main.rs" @@ -1 +1,78 @@ # aip-filter + +Parses and evaluates [AIP-160](https://google.aip.dev/160) filter expressions. + +## Install + +Clone the repository and install it with Cargo: + +```bash +cargo install --path . +``` + +## CLI usage + +```bash +usage: aip-filter [-p|--print] <expr> [file...] +``` + +* `--print` prints the parsed expression instead of evaluating it. +* If no files are given, reads from stdin. + +### Examples + +```bash +# Match if price is between 10 and 100, and status is active +echo '{"price": 93, "status": "active"}' | \ + aip-filter "price > 10 AND price < 100 AND status = active" + +# Match if 'foobar' appears in any field (bare-literal match) +echo '{"id": 1, "msg": "hello foobar"}' | aip-filter "foobar" + +# Match if 'tags' contains 'beta' +echo '{"tags": ["stable", "beta"]}' | aip-filter "tags:beta" + +# Nested field access +echo '{"user": {"settings": {"theme": "dark"}}}' | \ + aip-filter "user.settings.theme = dark" +``` + +## API + +Implement `Filterable` for your type, then call `Expr::evaluate`: + +```rust +use aip_filter::{parse, Value, Filterable}; + +struct Issue { + state: String, + priority: i64, + labels: Vec<String>, +} + +impl Filterable for Issue { + fn field(&self, name: &str) -> Option<Value<'_>> { + match name { + "state" => Some(Value::String(&self.state)), + "priority" => Some(Value::Int(self.priority)), + "labels" => Some(Value::List( + self.labels.iter().map(|l| Value::String(l)).collect() + )), + _ => None, + } + } +} + +fn main() { + let expr = parse("state = \"open\"").unwrap().unwrap(); + let issue = Issue { state: "open".into(), priority: 1, labels: vec![] }; + + if expr.evaluate(&issue) { + println!("got match"); + } else { + println!("no match"); + } + + assert!(expr.evaluate(&issue)); +} +``` diff --git a/src/ast.rs b/src/ast.rs new file mode 100644 index 0000000..54128d8 --- /dev/null +++ b/src/ast.rs @@ -0,0 +1,106 @@ +/// Position in the filter string (line and column, 1-indexed). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Position { + pub line: usize, + pub col: usize, +} + +impl std::fmt::Display for Position { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}:{}", self.line, self.col) + } +} + +/// Binary operators: AND / OR. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinaryOp { + And, + Or, +} + +/// Unary operators: negation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnaryOp { + Minus, + Not, +} + +/// Comparison operators. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CompareOp { + Equals, + NotEquals, + LessThan, + LessThanEquals, + GreaterThan, + GreaterThanEquals, + Has, + MatchesRegexp, + NotMatchesRegexp, +} + +impl CompareOp { + pub fn as_str(&self) -> &'static str { + match self { + CompareOp::Equals => "=", + CompareOp::NotEquals => "!=", + CompareOp::LessThan => "<", + CompareOp::LessThanEquals => "<=", + CompareOp::GreaterThan => ">", + CompareOp::GreaterThanEquals => ">=", + CompareOp::Has => ":", + CompareOp::MatchesRegexp => "=~", + CompareOp::NotMatchesRegexp => "!~", + } + } +} + +/// The AST of a parsed AIP-160 filter expression. +#[derive(Debug, Clone, PartialEq)] +pub enum Expr { + /// Binary expression: `left AND right` or `left OR right`. + Binary { + op: BinaryOp, + left: Box<Expr>, + right: Box<Expr>, + pos: Position, + }, + + /// Unary expression: `-expr` or `NOT expr`. + Unary { + op: UnaryOp, + expr: Box<Expr>, + pos: Position, + }, + + /// Comparison expression: `left op right`. + Comparison { + op: CompareOp, + left: Box<Expr>, + right: Box<Expr>, + pos: Position, + }, + + /// A simple name or literal value. + Name { + name: String, + pos: Position, + /// `true` if this was a quoted string literal. + is_string: bool, + /// `true` if inside a parenthesized (composite) expression. + is_composite: bool, + }, + + /// A member access expression: `holder.member`. + Member { + holder: Box<Expr>, + member: String, + pos: Position, + }, + + /// A function call expression: `fn(args...)`. + Function { + func: Box<Expr>, + args: Vec<Expr>, + }, +} diff --git a/src/display.rs b/src/display.rs new file mode 100644 index 0000000..f89f284 --- /dev/null +++ b/src/display.rs @@ -0,0 +1,126 @@ +use super::ast::{BinaryOp, Expr, UnaryOp}; +use std::fmt; + +impl fmt::Display for Expr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.print(f, 0) + } +} + +impl Expr { + fn print(&self, f: &mut fmt::Formatter<'_>, indent: usize) -> fmt::Result { + match self { + Expr::Binary { op, left, right, .. } => { + let label = match op { + BinaryOp::And => "conjunction", + BinaryOp::Or => "disjunction", + }; + writeln!(f, "{:indent$}{}", "", label, indent = indent)?; + left.print(f, indent + 2)?; + right.print(f, indent + 2) + } + + Expr::Unary { op, expr, .. } => { + let label = match op { + UnaryOp::Minus => "minus", + UnaryOp::Not => "not", + }; + writeln!(f, "{:indent$}{}", "", label, indent = indent)?; + expr.print(f, indent + 2) + } + + Expr::Comparison { op, left, right, .. } => { + writeln!(f, "{:indent$}compare {}", "", op.as_str(), indent = indent)?; + left.print(f, indent + 2)?; + right.print(f, indent + 2) + } + + Expr::Name { name, is_string, .. } => { + if *is_string { + writeln!(f, "{:indent$}{:?}", "", name, indent = indent) + } else { + writeln!(f, "{:indent$}{}", "", name, indent = indent) + } + } + + Expr::Member { holder, member, .. } => { + // Walk the member chain to collect all name segments. + let mut parts = vec![member.as_str()]; + let mut cur = holder.as_ref(); + loop { + match cur { + Expr::Member { holder, member, .. } => { + parts.push(member.as_str()); + cur = holder.as_ref(); + } + Expr::Name { name, .. } => { + parts.push(name.as_str()); + break; + } + _ => break, + } + } + parts.reverse(); + write!(f, "{:indent$}", "", indent = indent)?; + for (i, part) in parts.iter().enumerate() { + if i > 0 { + write!(f, ".")?; + } + write!(f, "{}", part)?; + } + writeln!(f) + } + + Expr::Function { func, args } => { + // "call" runs directly into the function name (no newline between them). + // The function name is indented at indent+2, which produces the + // two-space gap: "call funcname\n". + write!(f, "{:indent$}call", "", indent = indent)?; + func.print(f, indent + 2)?; + for arg in args { + arg.print(f, indent + 2)?; + } + Ok(()) + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::parse; + + #[test] + fn test_display_comparison() { + let expr = parse("x = 42").unwrap().unwrap(); + assert_eq!(format!("{expr}"), "compare =\n x\n 42\n"); + } + + #[test] + fn test_display_conjunction() { + let expr = parse("a : b c = 17").unwrap().unwrap(); + assert_eq!( + format!("{expr}"), + "conjunction\n compare :\n a\n b\n compare =\n c\n 17\n" + ); + } + + #[test] + fn test_display_member() { + let expr = parse("a.b > 17").unwrap().unwrap(); + assert_eq!(format!("{expr}"), "compare >\n a.b\n 17\n"); + } + + #[test] + fn test_display_function() { + let expr = parse("func(val)").unwrap().unwrap(); + assert_eq!(format!("{expr}"), "call func\n val\n"); + } + + #[test] + fn test_display_string_literal() { + // String literals are displayed with Rust's {:?} quoting. + let expr = parse(r#"x = "hello""#).unwrap().unwrap(); + assert_eq!(format!("{expr}"), "compare =\n x\n \"hello\"\n"); + } +} diff --git a/src/eval.rs b/src/eval.rs new file mode 100644 index 0000000..bf2afec --- /dev/null +++ b/src/eval.rs @@ -0,0 +1,920 @@ +use super::ast::{BinaryOp, CompareOp, Expr, UnaryOp}; +use regex::Regex; +use std::cmp::Ordering; + +/// A runtime value used during filter evaluation. +#[derive(Debug, Clone)] +pub enum Value<'a> { + /// Boolean value. + Bool(bool), + /// Signed integer (covers i8..i64). + Int(i64), + /// Unsigned integer (covers u8..u64). + Uint(u64), + /// Floating-point number. + Float(f64), + /// Borrowed string reference. + String(&'a str), + /// Owned string (for computed values). + StringOwned(std::string::String), + /// A list of values (for repeated fields / slices). + List(Vec<Value<'a>>), + /// A map of string keys to values. + Map(Vec<(&'a str, Value<'a>)>), + /// Absent value. + Null, +} + +impl<'a> Value<'a> { + /// Returns the string content if this is a string variant. + pub fn as_str(&self) -> Option<&str> { + match self { + Value::String(s) => Some(s), + Value::StringOwned(s) => Some(s.as_str()), + _ => None, + } + } + + /// Returns true if this value is the zero/default for its type. + pub fn is_zero(&self) -> bool { + match self { + Value::Bool(b) => !b, + Value::Int(i) => *i == 0, + Value::Uint(u) => *u == 0, + Value::Float(f) => *f == 0.0, + Value::String(s) => s.is_empty(), + Value::StringOwned(s) => s.is_empty(), + Value::List(v) => v.is_empty(), + Value::Map(m) => m.is_empty(), + Value::Null => true, + } + } + + /// Try to compare two values, returning an ordering if possible. + fn compare(&self, other: &Value<'_>) -> Option<Ordering> { + match (self, other) { + (Value::Bool(a), Value::Bool(b)) => Some(a.cmp(b)), + (Value::Int(a), Value::Int(b)) => Some(a.cmp(b)), + (Value::Uint(a), Value::Uint(b)) => Some(a.cmp(b)), + (Value::Float(a), Value::Float(b)) => a.partial_cmp(b), + (Value::String(a), Value::String(b)) => Some(a.to_lowercase().cmp(&b.to_lowercase())), + (Value::StringOwned(a), Value::String(b)) => { + Some(a.to_lowercase().cmp(&b.to_lowercase())) + } + (Value::String(a), Value::StringOwned(b)) => { + Some(a.to_lowercase().cmp(&b.to_lowercase())) + } + (Value::StringOwned(a), Value::StringOwned(b)) => { + Some(a.to_lowercase().cmp(&b.to_lowercase())) + } + // Cross-numeric comparisons + (Value::Int(a), Value::Float(b)) => (*a as f64).partial_cmp(b), + (Value::Float(a), Value::Int(b)) => a.partial_cmp(&(*b as f64)), + (Value::Uint(a), Value::Float(b)) => (*a as f64).partial_cmp(b), + (Value::Float(a), Value::Uint(b)) => a.partial_cmp(&(*b as f64)), + (Value::Int(a), Value::Uint(b)) => { + if *a < 0 { + Some(Ordering::Less) + } else { + (*a as u64).partial_cmp(b) + } + } + (Value::Uint(a), Value::Int(b)) => { + if *b < 0 { + Some(Ordering::Greater) + } else { + a.partial_cmp(&(*b as u64)) + } + } + _ => None, + } + } +} + +/// Trait that types must implement to be filterable. +/// +/// # Example +/// +/// ```rust +/// use aip_filter::{Value, Filterable}; +/// +/// struct Issue { +/// state: String, +/// priority: i64, +/// } +/// +/// impl Filterable for Issue { +/// fn field(&self, name: &str) -> Option<Value<'_>> { +/// match name { +/// "state" => Some(Value::String(&self.state)), +/// "priority" => Some(Value::Int(self.priority)), +/// _ => None, +/// } +/// } +/// } +/// ``` +pub trait Filterable { + /// Look up a field by name, returning its value. + /// + /// The name matching should ideally support both snake_case and camelCase + /// forms, though this is up to the implementor. + /// + /// Return `None` if the field doesn't exist. + fn field(&self, name: &str) -> Option<Value<'_>>; + + /// Look up a nested field path like `["a", "b"]` for `a.b`. + /// + /// Default implementation walks through `field()` calls for structs. + /// Override this for custom nested access. + fn field_path(&self, path: &[&str]) -> Option<Value<'_>> { + if path.is_empty() { + return None; + } + if path.len() == 1 { + return self.field(path[0]); + } + // Default: can't traverse deeper without knowing the sub-type + None + } + + /// Return all field values for "global restriction" matching + /// (when a bare value appears without a field name). + /// + /// Default returns an empty vec (no global matching). + fn all_field_values(&self) -> Vec<Value<'_>> { + Vec::new() + } +} + +impl Expr { + /// Evaluate this expression against a filterable value. + /// + /// Returns `true` if the value matches the filter. + pub fn evaluate<T: Filterable>(&self, val: &T) -> bool { + match self { + Expr::Binary { op, left, right, .. } => match op { + BinaryOp::And => left.evaluate(val) && right.evaluate(val), + BinaryOp::Or => left.evaluate(val) || right.evaluate(val), + }, + + Expr::Unary { op, expr, .. } => { + let result = expr.evaluate(val); + match op { + UnaryOp::Minus | UnaryOp::Not => !result, + } + } + + Expr::Comparison { + op, left, right, .. + } => eval_comparison(*op, left, right, val), + + Expr::Name { + name, is_string, .. + } => { + // A bare name/value: global restriction matching. + // Check if any field contains or equals the value. + if let Some(field_val) = val.field(name) { + // If it's a boolean field referenced by name, return its value + if !*is_string + && let Value::Bool(b) = field_val { + return b; + } + return !field_val.is_zero(); + } + + // Try matching against all fields + let search = name.as_str(); + for field_val in val.all_field_values() { + if value_contains_str(&field_val, search) { + return true; + } + } + false + } + + Expr::Member { + holder, member, .. + } => { + // Member as a standalone expression; expect boolean + if let Some(val) = resolve_member_value(holder, member, val) { + match val { + Value::Bool(b) => b, + _ => !val.is_zero(), + } + } else { + false + } + } + + Expr::Function { .. } => { + // Function evaluation is not yet supported + false + } + } + } +} + +/// Evaluate a comparison expression. +fn eval_comparison<T: Filterable>( + op: CompareOp, + left: &Expr, + right: &Expr, + val: &T, +) -> bool { + // Resolve the left side to a Value + let left_val = resolve_expr_value(left, val); + + let Some(left_val) = left_val else { + return false; + }; + + eval_comparison_with_value(op, &left_val, right, val) +} + +fn eval_comparison_with_value<T: Filterable>( + op: CompareOp, + left_val: &Value<'_>, + right: &Expr, + val: &T, +) -> bool { + // Handle the right side, it could be a composite expression (AND/OR) + match right { + Expr::Binary { + op: bin_op, + left: rleft, + right: rright, + .. + } => { + let l = eval_comparison_with_value(op, left_val, rleft, val); + let r = eval_comparison_with_value(op, left_val, rright, val); + match bin_op { + BinaryOp::And => l && r, + BinaryOp::Or => l || r, + } + } + Expr::Unary { + op: UnaryOp::Minus | UnaryOp::Not, + expr, + .. + } => !eval_comparison_with_value(op, left_val, expr, val), + + _ => { + let right_str = expr_to_literal_str(right); + let right_str = right_str.as_deref(); + + match (&left_val, op) { + // Has operator on lists + (Value::List(items), CompareOp::Has) => { + if let Some(rs) = right_str { + items.iter().any(|item| value_matches_str(item, rs, op)) + } else { + false + } + } + + // Has operator on maps; check if key exists + (Value::Map(entries), CompareOp::Has) => { + if let Some(rs) = right_str { + entries.iter().any(|(k, _)| k.eq_ignore_ascii_case(rs)) + } else { + false + } + } + + // Equals/NotEquals on lists; check if any element matches + (Value::List(items), CompareOp::Equals) => { + if let Some(rs) = right_str { + items.iter().any(|item| value_matches_str(item, rs, op)) + } else { + false + } + } + (Value::List(items), CompareOp::NotEquals) => { + if let Some(rs) = right_str { + !items.iter().any(|item| value_matches_str(item, rs, CompareOp::Equals)) + } else { + true + } + } + + // Has on strings; substring/word boundary match + (_, CompareOp::Has) => { + if let Some(rs) = right_str { + // Wildcard handling + if rs == "*" { + return !left_val.is_zero(); + } + value_matches_str(left_val, rs, op) + } else { + false + } + } + + // Regex match + (_, CompareOp::MatchesRegexp | CompareOp::NotMatchesRegexp) => { + let Some(left_s) = left_val.as_str() else { + return false; + }; + let Some(rs) = right_str else { + return false; + }; + let Ok(re) = Regex::new(rs) else { + return false; + }; + let matched = re.is_match(left_s); + if op == CompareOp::NotMatchesRegexp { + !matched + } else { + matched + } + } + + // Standard comparison + _ => { + if let Some(rs) = right_str { + let right_val = parse_literal_for_value(left_val, rs); + compare_values(op, left_val, &right_val) + } else { + false + } + } + } + } + } +} + +/// Resolve an expression to its value given a filterable object. +fn resolve_expr_value<'a, T: Filterable>(expr: &Expr, val: &'a T) -> Option<Value<'a>> { + match expr { + Expr::Name { + name, is_string, .. + } => { + if *is_string { + // A quoted string on the left side of a comparison is just a string literal + Some(Value::StringOwned(name.clone())) + } else { + val.field(name) + } + } + Expr::Member { + holder, member, .. + } => resolve_member_value(holder, member, val), + _ => None, + } +} + +/// Resolve a member expression (e.g., `a.b.c`) to a value. +fn resolve_member_value<'a, T: Filterable>( + holder: &Expr, + member: &str, + val: &'a T, +) -> Option<Value<'a>> { + // Collect the path segments + let mut path = vec![member]; + let mut cur = holder; + loop { + match cur { + Expr::Member { + holder, + member, + .. + } => { + path.push(member.as_str()); + cur = holder; + } + Expr::Name { name, .. } => { + path.push(name.as_str()); + break; + } + _ => return None, + } + } + path.reverse(); + + // Try field_path first + if let Some(v) = val.field_path(&path) { + return Some(v); + } + + // Fall back to walking field by field (for nested Filterable values) + // First field + let first_val = val.field(path[0])?; + + // Walk remaining path through map keys + let mut current = first_val; + for &segment in &path[1..] { + current = match current { + Value::Map(entries) => { + let found = entries + .into_iter() + .find(|(k, _)| k.eq_ignore_ascii_case(segment)); + match found { + Some((_, v)) => v, + None => return None, + } + } + _ => return None, + }; + } + Some(current) +} + +/// Extract the literal string from an expression (name or member chain as dotted string). +fn expr_to_literal_str(expr: &Expr) -> Option<String> { + match expr { + Expr::Name { name, .. } => Some(name.clone()), + Expr::Member { holder, member, .. } => { + // a.b on the right side is just "a.b" as a string + let mut parts = vec![member.as_str()]; + let mut cur = holder.as_ref(); + loop { + match cur { + Expr::Member { holder, member, .. } => { + parts.push(member.as_str()); + cur = holder.as_ref(); + } + Expr::Name { name, .. } => { + parts.push(name.as_str()); + break; + } + _ => return None, + } + } + parts.reverse(); + Some(parts.join(".")) + } + Expr::Function { .. } => None, + _ => None, + } +} + +/// Parse a literal string to match the type of a reference value. +fn parse_literal_for_value<'a>(reference: &Value<'_>, literal: &'a str) -> Value<'a> { + match reference { + Value::Bool(_) => match literal.to_lowercase().as_str() { + "true" | "1" => Value::Bool(true), + "false" | "0" => Value::Bool(false), + _ => Value::String(literal), + }, + Value::Int(_) => { + if let Ok(i) = literal.parse::<i64>() { + Value::Int(i) + } else if let Ok(f) = literal.parse::<f64>() { + Value::Int(f as i64) + } else { + Value::String(literal) + } + } + Value::Uint(_) => { + if let Ok(u) = literal.parse::<u64>() { + Value::Uint(u) + } else { + Value::String(literal) + } + } + Value::Float(_) => { + if let Ok(f) = literal.parse::<f64>() { + Value::Float(f) + } else { + Value::String(literal) + } + } + Value::String(_) | Value::StringOwned(_) => Value::String(literal), + _ => Value::String(literal), + } +} + +/// Check if a value matches a string using the given comparison op. +fn value_matches_str(val: &Value<'_>, s: &str, op: CompareOp) -> bool { + match val { + Value::String(vs) => string_match(vs, s, op), + Value::StringOwned(vs) => string_match(vs.as_str(), s, op), + Value::Int(i) => { + if let Ok(ri) = s.parse::<i64>() { + match op { + CompareOp::Equals | CompareOp::Has => *i == ri, + _ => false, + } + } else { + false + } + } + Value::Uint(u) => { + if let Ok(ru) = s.parse::<u64>() { + match op { + CompareOp::Equals | CompareOp::Has => *u == ru, + _ => false, + } + } else { + false + } + } + Value::Float(f) => { + if let Ok(rf) = s.parse::<f64>() { + match op { + CompareOp::Equals | CompareOp::Has => (*f - rf).abs() < f64::EPSILON, + _ => false, + } + } else { + false + } + } + Value::Bool(b) => { + let sb = matches!(s.to_lowercase().as_str(), "true" | "1"); + match op { + CompareOp::Equals | CompareOp::Has => *b == sb, + _ => false, + } + } + Value::List(items) => items.iter().any(|item| value_matches_str(item, s, op)), + Value::Map(entries) => entries.iter().any(|(_, v)| value_matches_str(v, s, op)), + Value::Null => false, + } +} + +/// String matching: case-insensitive equality for `=`, word-boundary match for `:`, wildcard support. +fn string_match(haystack: &str, needle: &str, op: CompareOp) -> bool { + match op { + CompareOp::Has => { + // Wildcard: * prefix + if let Some(suffix) = needle.strip_prefix('*') { + return haystack.len() >= suffix.len() + && haystack[haystack.len() - suffix.len()..].eq_ignore_ascii_case(suffix); + } + // Wildcard: * suffix + if let Some(prefix) = needle.strip_suffix('*') { + return haystack.len() >= prefix.len() + && haystack[..prefix.len()].eq_ignore_ascii_case(prefix); + } + + // Word boundary match + let hl = haystack.to_lowercase(); + let nl = needle.to_lowercase(); + if let Some(idx) = hl.find(&nl) { + let is_boundary = |b: u8| -> bool { + b < 0x80 && !b.is_ascii_alphanumeric() + }; + let start_ok = idx == 0 || is_boundary(haystack.as_bytes()[idx - 1]); + let end_ok = idx + nl.len() >= haystack.len() + || is_boundary(haystack.as_bytes()[idx + nl.len()]); + if start_ok && end_ok { + return true; + } + } + + // Also check individual words + let words = split_string(haystack); + if words.len() > 1 { + for word in &words { + if string_match(word, needle, CompareOp::Has) { + return true; + } + } + } + + false + } + CompareOp::Equals => { + // Wildcard: * prefix + if let Some(suffix) = needle.strip_prefix('*') { + return haystack.len() >= suffix.len() + && haystack[haystack.len() - suffix.len()..].eq_ignore_ascii_case(suffix); + } + // Wildcard: * suffix + if let Some(prefix) = needle.strip_suffix('*') { + return haystack.len() >= prefix.len() + && haystack[..prefix.len()].eq_ignore_ascii_case(prefix); + } + haystack.eq_ignore_ascii_case(needle) + } + CompareOp::NotEquals => !haystack.eq_ignore_ascii_case(needle), + _ => { + // For ordering comparisons on strings, use case-sensitive compare + let ord = haystack.cmp(needle); + match op { + CompareOp::LessThan => ord == Ordering::Less, + CompareOp::LessThanEquals => ord != Ordering::Greater, + CompareOp::GreaterThan => ord == Ordering::Greater, + CompareOp::GreaterThanEquals => ord != Ordering::Less, + _ => false, + } + } + } +} + +/// Check if a value contains a string (for global restriction matching). +fn value_contains_str(val: &Value<'_>, search: &str) -> bool { + match val { + Value::String(s) => s.to_lowercase().contains(&search.to_lowercase()), + Value::StringOwned(s) => s.to_lowercase().contains(&search.to_lowercase()), + Value::Int(i) => i.to_string() == search, + Value::Uint(u) => u.to_string() == search, + Value::Float(f) => f.to_string() == search, + Value::Bool(b) => { + let bs = if *b { "true" } else { "false" }; + bs == search.to_lowercase() + } + Value::List(items) => items.iter().any(|item| value_contains_str(item, search)), + Value::Map(entries) => entries.iter().any(|(_, v)| value_contains_str(v, search)), + Value::Null => false, + } +} + +/// Compare two values using a comparison operator. +fn compare_values(op: CompareOp, left: &Value<'_>, right: &Value<'_>) -> bool { + match op { + CompareOp::Equals | CompareOp::Has => { + if let (Some(ls), Some(rs)) = (left.as_str(), right.as_str()) { + return string_match(ls, rs, op); + } + left.compare(right) == Some(Ordering::Equal) + } + CompareOp::NotEquals => { + if let (Some(ls), Some(rs)) = (left.as_str(), right.as_str()) { + return !ls.eq_ignore_ascii_case(rs); + } + left.compare(right) != Some(Ordering::Equal) + } + CompareOp::LessThan => left.compare(right) == Some(Ordering::Less), + CompareOp::LessThanEquals => matches!( + left.compare(right), + Some(Ordering::Less) | Some(Ordering::Equal) + ), + CompareOp::GreaterThan => left.compare(right) == Some(Ordering::Greater), + CompareOp::GreaterThanEquals => matches!( + left.compare(right), + Some(Ordering::Greater) | Some(Ordering::Equal) + ), + CompareOp::MatchesRegexp | CompareOp::NotMatchesRegexp => { + // Handled at a higher level + false + } + } +} + +/// Split a string into words. +fn split_string(s: &str) -> Vec<&str> { + let mut result = Vec::new(); + let mut start = None; + + for (i, c) in s.char_indices() { + if c.is_whitespace() { + if let Some(s_idx) = start { + result.push(&s[s_idx..i]); + start = None; + } + } else if is_text_start_char(c) { + if start.is_none() { + start = Some(i); + } + } else { + if let Some(s_idx) = start { + result.push(&s[s_idx..i]); + start = None; + } + // Single char becomes its own word + let end = i + c.len_utf8(); + result.push(&s[i..end]); + } + } + + if let Some(s_idx) = start { + result.push(&s[s_idx..]); + } + + result +} + +fn is_text_start_char(r: char) -> bool { + matches!(r, + '#'..='\'' | '*' | '/' | ';' | '?' | '@' | 'A'..='Z' | '[' | ']' | '^'..='}' | '\\' + ) || ('\u{00a1}'..='\u{0effff}').contains(&r) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parse; + + struct TestItem { + state: String, + priority: i64, + score: f64, + active: bool, + labels: Vec<String>, + name: String, + } + + impl Filterable for TestItem { + fn field(&self, name: &str) -> Option<Value<'_>> { + match name { + "state" => Some(Value::String(&self.state)), + "priority" => Some(Value::Int(self.priority)), + "score" => Some(Value::Float(self.score)), + "active" => Some(Value::Bool(self.active)), + "labels" => Some(Value::List( + self.labels.iter().map(|l| Value::String(l.as_str())).collect(), + )), + "name" => Some(Value::String(&self.name)), + _ => None, + } + } + + fn all_field_values(&self) -> Vec<Value<'_>> { + vec![ + Value::String(&self.state), + Value::Int(self.priority), + Value::Float(self.score), + Value::Bool(self.active), + Value::String(&self.name), + ] + } + } + + fn item() -> TestItem { + TestItem { + state: "open".into(), + priority: 5, + score: 9.5, + active: true, + labels: vec!["bug".into(), "urgent".into()], + name: "Fix crash on startup".into(), + } + } + + #[test] + fn test_equals() { + let expr = parse(r#"state = "open""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"state = "closed""#).unwrap().unwrap(); + assert!(!expr.evaluate(&item())); + } + + #[test] + fn test_not_equals() { + let expr = parse(r#"state != "closed""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_numeric_comparison() { + let expr = parse("priority > 3").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse("priority < 3").unwrap().unwrap(); + assert!(!expr.evaluate(&item())); + + let expr = parse("priority >= 5").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse("priority <= 5").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_float_comparison() { + let expr = parse("score > 9.0").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_and() { + let expr = parse(r#"state = "open" AND priority > 3"#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"state = "closed" AND priority > 3"#) + .unwrap() + .unwrap(); + assert!(!expr.evaluate(&item())); + } + + #[test] + fn test_or() { + let expr = parse(r#"state = "closed" OR priority > 3"#) + .unwrap() + .unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_negation() { + let expr = parse(r#"-state = "closed""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"NOT state = "open""#).unwrap().unwrap(); + assert!(!expr.evaluate(&item())); + } + + #[test] + fn test_has_on_list() { + let expr = parse(r#"labels : "bug""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"labels : "feature""#).unwrap().unwrap(); + assert!(!expr.evaluate(&item())); + } + + #[test] + fn test_has_on_string() { + let expr = parse(r#"name : "crash""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"name : "memory""#).unwrap().unwrap(); + assert!(!expr.evaluate(&item())); + } + + #[test] + fn test_boolean_field() { + let expr = parse("active").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_regex_match() { + let expr = parse(r#"state =~ "^op""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"state !~ "^cl""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_wildcard() { + let expr = parse(r#"state = "op*""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse(r#"state = "*en""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_case_insensitive() { + let expr = parse(r#"state = "OPEN""#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_implicit_and() { + let expr = parse(r#"state = "open" priority > 3"#).unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_parenthesized() { + let expr = parse(r#"(state = "open" OR state = "closed") AND priority > 3"#) + .unwrap() + .unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_has_wildcard_star() { + let expr = parse("priority : *").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + } + + #[test] + fn test_global_restriction() { + let expr = parse("open").unwrap().unwrap(); + assert!(expr.evaluate(&item())); + + let expr = parse("nonexistent").unwrap().unwrap(); + assert!(!expr.evaluate(&item())); + } + + #[test] + fn test_nested_field() { + struct Outer { + inner: Inner, + } + struct Inner { + value: i64, + } + + impl Filterable for Outer { + fn field(&self, name: &str) -> Option<Value<'_>> { + match name { + "inner" => Some(Value::Map(vec![ + ("value", Value::Int(self.inner.value)), + ])), + _ => None, + } + } + + fn field_path(&self, path: &[&str]) -> Option<Value<'_>> { + match path { + ["inner", "value"] => Some(Value::Int(self.inner.value)), + _ => None, + } + } + } + + let outer = Outer { + inner: Inner { value: 42 }, + }; + + let expr = parse("inner.value > 10").unwrap().unwrap(); + assert!(expr.evaluate(&outer)); + } +} diff --git a/src/lex.rs b/src/lex.rs new file mode 100644 index 0000000..284fd75 --- /dev/null +++ b/src/lex.rs @@ -0,0 +1,574 @@ +use super::ast::Position; + +/// Token kinds produced by the lexer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TokenKind { + InvalidInput, + Eof, + Wsc, // whitespace or comment + Dot, // . + Has, // : + Or, // OR + And, // AND + Not, // NOT + Lparen, // ( + Rparen, // ) + Comma, // , + LessThan, // < + GreaterThan, // > + GreaterThanEquals, // >= + LessThanEquals, // <= + NotEquals, // != + MatchesRegexp, // =~ + NotMatchesRegexp, // !~ + Equals, // = + Minus, // - + Plus, // + + Tilde, // ~ + #[allow(dead_code)] + Backslash, // backslash + StringLit, // quoted string + Text, // unquoted text/name +} + +/// A lexical token. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Token { + pub kind: TokenKind, + pub pos: Position, + pub val: String, // only set for StringLit and Text +} + +/// Lexer converts a filter string into tokens. +pub(crate) struct Lexer { + input: Vec<char>, + cursor: usize, + pushed: Option<Token>, + prev_dot: bool, + prev_text: bool, + next_pos: Position, +} + +impl Lexer { + pub fn new(input: &str) -> Self { + Lexer { + input: input.chars().collect(), + cursor: 0, + pushed: None, + prev_dot: false, + prev_text: false, + next_pos: Position { line: 1, col: 0 }, + } + } + + pub fn next_token(&mut self) -> Token { + if let Some(tok) = self.pushed.take() { + return tok; + } + + let pos = self.next_pos; + let prev_dot = self.prev_dot; + let prev_text = self.prev_text; + self.prev_dot = false; + self.prev_text = false; + + let Some(r) = self.peek() else { + return Token { kind: TokenKind::Eof, pos, val: String::new() }; + }; + + self.advance(); + + match r { + _ if is_white(r) => { + self.skip_white(); + Token { kind: TokenKind::Wsc, pos, val: String::new() } + } + '-' => { + if let Some(rn) = self.peek() { + if rn == '-' { + self.advance(); + self.skip_comment(); + self.skip_white(); + return Token { kind: TokenKind::Wsc, pos, val: String::new() }; + } + if is_digit(rn) { + return self.text(r, prev_dot); + } + if !prev_text && rn == '.' + && let Some(&next_after) = self.input.get(self.cursor + 1) + && is_digit(next_after) { + return self.text(r, prev_dot); + } + } + Token { kind: TokenKind::Minus, pos, val: String::new() } + } + '.' => { + if self.peek().is_none() { + self.prev_dot = true; + return Token { kind: TokenKind::Dot, pos, val: String::new() }; + } + if !prev_text + && let Some(rn) = self.peek() + && is_digit(rn) { + return self.text(r, prev_dot); + } + self.prev_dot = true; + Token { kind: TokenKind::Dot, pos, val: String::new() } + } + ':' => Token { kind: TokenKind::Has, pos, val: String::new() }, + 'O' => { + if self.remaining() > 0 && self.input[self.cursor] == 'R' + && (self.remaining() < 2 || is_white(self.input[self.cursor + 1])) + { + self.advance(); + return Token { kind: TokenKind::Or, pos, val: String::new() }; + } + self.text(r, prev_dot) + } + 'A' => { + if self.remaining() > 1 + && self.input[self.cursor] == 'N' + && self.input[self.cursor + 1] == 'D' + && (self.remaining() < 3 || is_white(self.input[self.cursor + 2])) + { + self.advance(); + self.advance(); + return Token { kind: TokenKind::And, pos, val: String::new() }; + } + self.text(r, prev_dot) + } + 'N' => { + if self.remaining() > 1 + && self.input[self.cursor] == 'O' + && self.input[self.cursor + 1] == 'T' + && (self.remaining() < 3 || is_white(self.input[self.cursor + 2])) + { + self.advance(); + self.advance(); + return Token { kind: TokenKind::Not, pos, val: String::new() }; + } + self.text(r, prev_dot) + } + '(' => Token { kind: TokenKind::Lparen, pos, val: String::new() }, + ')' => Token { kind: TokenKind::Rparen, pos, val: String::new() }, + ',' => Token { kind: TokenKind::Comma, pos, val: String::new() }, + '<' => { + if self.peek() == Some('=') { + self.advance(); + Token { kind: TokenKind::LessThanEquals, pos, val: String::new() } + } else { + Token { kind: TokenKind::LessThan, pos, val: String::new() } + } + } + '>' => { + if self.peek() == Some('=') { + self.advance(); + Token { kind: TokenKind::GreaterThanEquals, pos, val: String::new() } + } else { + Token { kind: TokenKind::GreaterThan, pos, val: String::new() } + } + } + '!' => { + if let Some(rn) = self.peek() { + if rn == '=' { + self.advance(); + return Token { kind: TokenKind::NotEquals, pos, val: String::new() }; + } + if rn == '~' { + self.advance(); + return Token { kind: TokenKind::NotMatchesRegexp, pos, val: String::new() }; + } + } + // bare ! is part of a name + self.text(r, prev_dot) + } + '=' => { + if self.peek() == Some('~') { + self.advance(); + Token { kind: TokenKind::MatchesRegexp, pos, val: String::new() } + } else { + Token { kind: TokenKind::Equals, pos, val: String::new() } + } + } + '+' => Token { kind: TokenKind::Plus, pos, val: String::new() }, + '~' => Token { kind: TokenKind::Tilde, pos, val: String::new() }, + '"' => self.collect_string(), + _ if is_text_start(r) || is_digit(r) => self.text(r, prev_dot), + _ => Token { kind: TokenKind::InvalidInput, pos, val: String::new() }, + } + } + + pub fn push_token(&mut self, tok: Token) { + assert!(self.pushed.is_none(), "double push_token"); + self.pushed = Some(tok); + } + + // Helpers. + + fn peek(&self) -> Option<char> { + self.input.get(self.cursor).copied() + } + + fn remaining(&self) -> usize { + self.input.len() - self.cursor + } + + fn advance(&mut self) -> Option<char> { + if self.cursor >= self.input.len() { + return None; + } + let r = self.input[self.cursor]; + self.cursor += 1; + if r == '\n' { + self.next_pos.line += 1; + self.next_pos.col = 0; + } else { + self.next_pos.col += 1; + } + Some(r) + } + + fn skip_white(&mut self) { + loop { + match self.peek() { + Some(r) if is_white(r) => { self.advance(); } + Some('-') => { + if self.input.get(self.cursor + 1) == Some(&'-') { + self.advance(); + self.advance(); + self.skip_comment(); + } else { + return; + } + } + _ => return, + } + } + } + + fn skip_comment(&mut self) { + while let Some(r) = self.peek() { + if r == '\t' || (' '..='~').contains(&r) || is_a1_or_higher(r) { + self.advance(); + } else { + return; + } + } + } + + fn text(&mut self, first: char, prev_dot: bool) -> Token { + let pos = self.next_pos; + + let mut sb = String::new(); + + if first != '\\' { + sb.push(first); + } else if !self.text_esc(&mut sb) { + return Token { kind: TokenKind::InvalidInput, pos, val: String::new() }; + } + + // Number prefix handling + if first == '-' || first == '.' || is_digit(first) { + let mut r = first; + while let Some(rn) = self.peek() { + if is_digit(rn) { + sb.push(rn); + self.advance(); + if r == '.' { + break; + } + } else if !prev_dot && r != '.' && rn == '.' { + sb.push(rn); + self.advance(); + break; + } else { + break; + } + if r == '-' { + r = rn; + } + } + } + + loop { + let Some(rn) = self.peek() else { break }; + if rn == '\\' { + self.advance(); + if !self.text_esc(&mut sb) { + return Token { kind: TokenKind::InvalidInput, pos, val: String::new() }; + } + } else if is_text_start(rn) || is_digit(rn) || rn == '+' || rn == '-' { + sb.push(rn); + self.advance(); + } else if rn == '!' { + // ! is part of a name unless followed by = or ~ + if let Some(&next) = self.input.get(self.cursor + 1) + && (next == '=' || next == '~') { + break; + } + sb.push(rn); + self.advance(); + } else { + break; + } + } + + self.prev_text = true; + Token { kind: TokenKind::Text, pos, val: sb } + } + + fn text_esc(&mut self, sb: &mut String) -> bool { + let Some(r) = self.peek() else { return false }; + self.handle_text_esc(r, sb) + } + + fn handle_text_esc(&mut self, r: char, sb: &mut String) -> bool { + match r { + ',' | ':' | '=' | '<' | '>' | '+' | '~' | '"' | '\\' | '.' | '*' => { + self.advance(); + sb.push(r); + true + } + 'u' => self.handle_hex(r, sb), + '0'..='7' => { + // Octal escapes + let d0 = r as u32 - '0' as u32; + if r <= '3' && self.remaining() >= 3 { + let n1 = self.input[self.cursor + 1]; + let n2 = self.input[self.cursor + 2]; + if is_octal_digit(n1) && is_octal_digit(n2) { + self.advance(); // skip r + self.advance(); // skip n1 + self.advance(); // skip n2 + let val = (d0 << 6) + + ((n1 as u32 - '0' as u32) << 3) + + (n2 as u32 - '0' as u32); + if let Some(c) = char::from_u32(val) { + sb.push(c); + } else { + sb.push(char::REPLACEMENT_CHARACTER); + } + return true; + } + } + if self.remaining() >= 2 { + let n1 = self.input[self.cursor + 1]; + if is_octal_digit(n1) { + self.advance(); // skip r + self.advance(); // skip n1 + let val = (d0 << 3) + (n1 as u32 - '0' as u32); + if let Some(c) = char::from_u32(val) { + sb.push(c); + } + return true; + } + } + self.advance(); // skip r + if let Some(c) = char::from_u32(d0) { + sb.push(c); + } + true + } + 'x' => self.handle_hex(r, sb), + _ => false, + } + } + + fn handle_hex(&mut self, starter: char, sb: &mut String) -> bool { + let len = if starter == 'u' { 4 } else { 2 }; + + if self.remaining() < len + 1 { + return false; + } + + // Check all hex digits (they start at cursor+1) + for i in 1..=len { + if !is_hex_digit(self.input[self.cursor + i]) { + return false; + } + } + + // Skip the starter ('u' or 'x') + self.advance(); + + let mut val: u32 = 0; + for _ in 0..len { + let r1 = self.input[self.cursor]; + self.advance(); + val <<= 4; + if ('a'..='f').contains(&r1) { + val += 10 + r1 as u32 - 'a' as u32; + } else if ('A'..='F').contains(&r1) { + val += 10 + r1 as u32 - 'A' as u32; + } else { + val += r1 as u32 - '0' as u32; + } + } + + if starter == 'u' { + if let Some(c) = char::from_u32(val) { + sb.push(c); + } else { + sb.push(char::REPLACEMENT_CHARACTER); + } + } else { + // \xHH: treat as the Unicode scalar value with that code point. + // All values 0x00..=0xFF are valid Unicode scalar values. + if let Some(c) = char::from_u32(val) { + sb.push(c); + } else { + sb.push(char::REPLACEMENT_CHARACTER); + } + } + true + } + + fn collect_string(&mut self) -> Token { + let start_pos = self.next_pos; + let mut sb = String::new(); + while let Some(r) = self.peek() { + self.advance(); + match r { + '"' => return Token { kind: TokenKind::StringLit, pos: start_pos, val: sb }, + '\\' => { + let Some(rn) = self.peek() else { + return Token { kind: TokenKind::InvalidInput, pos: start_pos, val: String::new() }; + }; + if !self.handle_text_esc(rn, &mut sb) { + // String-only single-char escape sequences not handled by handle_text_esc + match rn { + 'a' => { self.advance(); sb.push('\x07'); } + 'b' => { self.advance(); sb.push('\x08'); } + 'f' => { self.advance(); sb.push('\x0c'); } + 'n' => { self.advance(); sb.push('\n'); } + 'r' => { self.advance(); sb.push('\r'); } + 't' => { self.advance(); sb.push('\t'); } + 'v' => { self.advance(); sb.push('\x0b'); } + // Unrecognised escape: keep the backslash + _ => sb.push('\\'), + } + } + } + _ => sb.push(r), + } + } + // Unterminated string + Token { kind: TokenKind::InvalidInput, pos: start_pos, val: String::new() } + } +} + +fn is_white(r: char) -> bool { + matches!(r, ' ' | '\t' | '\x0c' | '\u{00a0}' | '\r' | '\n') +} + +fn is_digit(r: char) -> bool { + ('0'..='9').contains(&r) +} + +fn is_octal_digit(r: char) -> bool { + ('0'..='7').contains(&r) +} + +fn is_hex_digit(r: char) -> bool { + is_digit(r) || ('a'..='f').contains(&r) || ('A'..='F').contains(&r) +} + +fn is_text_start(r: char) -> bool { + matches!(r, + '#'..='\'' | '*' | '/' | ';' | '?' | '@' | 'A'..='Z' | '[' | ']' | '^'..='}' | '\\' + ) || is_a1_or_higher(r) +} + +fn is_a1_or_higher(r: char) -> bool { + ('\u{00a1}'..='\u{0effff}').contains(&r) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_basic_tokens() { + let mut lex = Lexer::new("x = 42"); + let t1 = lex.next_token(); + assert_eq!(t1.kind, TokenKind::Text); + assert_eq!(t1.val, "x"); + + let t2 = lex.next_token(); + assert_eq!(t2.kind, TokenKind::Wsc); + + let t3 = lex.next_token(); + assert_eq!(t3.kind, TokenKind::Equals); + + let t4 = lex.next_token(); + assert_eq!(t4.kind, TokenKind::Wsc); + + let t5 = lex.next_token(); + assert_eq!(t5.kind, TokenKind::Text); + assert_eq!(t5.val, "42"); + + let t6 = lex.next_token(); + assert_eq!(t6.kind, TokenKind::Eof); + } + + #[test] + fn test_keywords() { + let mut lex = Lexer::new("AND OR NOT"); + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::And); + lex.next_token(); // wsc + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::Or); + lex.next_token(); // wsc + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::Not); + } + + #[test] + fn test_string() { + let mut lex = Lexer::new(r#""hello world""#); + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::StringLit); + assert_eq!(t.val, "hello world"); + } + + #[test] + fn test_comparison_ops() { + let mut lex = Lexer::new("< <= > >= != =~ !~"); + assert_eq!(lex.next_token().kind, TokenKind::LessThan); + lex.next_token(); // wsc + assert_eq!(lex.next_token().kind, TokenKind::LessThanEquals); + lex.next_token(); + assert_eq!(lex.next_token().kind, TokenKind::GreaterThan); + lex.next_token(); + assert_eq!(lex.next_token().kind, TokenKind::GreaterThanEquals); + lex.next_token(); + assert_eq!(lex.next_token().kind, TokenKind::NotEquals); + lex.next_token(); + assert_eq!(lex.next_token().kind, TokenKind::MatchesRegexp); + lex.next_token(); + assert_eq!(lex.next_token().kind, TokenKind::NotMatchesRegexp); + } + + #[test] + fn test_comment() { + let mut lex = Lexer::new("x -- this is a comment\n= 42"); + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::Text); + assert_eq!(t.val, "x"); + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::Wsc); + let t = lex.next_token(); + assert_eq!(t.kind, TokenKind::Equals); + } + + #[test] + fn test_push_token() { + let mut lex = Lexer::new("a b"); + let t = lex.next_token(); + assert_eq!(t.val, "a"); + lex.push_token(t.clone()); + let t2 = lex.next_token(); + assert_eq!(t2.val, "a"); + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..60e05d3 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,59 @@ +//! Parses and evaluates [AIP-160](https://google.aip.dev/160) filter expressions. +//! +//! ## Example +//! +//! ```rust +//! use aip_filter::{parse, Value, Filterable}; +//! +//! let expr = parse("state = \"open\" AND priority > 3").unwrap().unwrap(); +//! println!("{}", expr); +//! ``` +//! +//! ## Evaluating filters +//! +//! Implement [`Filterable`] for your type, then call [`Expr::evaluate`]: +//! +//! ```rust +//! use aip_filter::{parse, Value, Filterable}; +//! +//! struct Issue { +//! state: String, +//! priority: i64, +//! labels: Vec<String>, +//! } +//! +//! impl Filterable for Issue { +//! fn field(&self, name: &str) -> Option<Value<'_>> { +//! match name { +//! "state" => Some(Value::String(&self.state)), +//! "priority" => Some(Value::Int(self.priority)), +//! "labels" => Some(Value::List( +//! self.labels.iter().map(|l| Value::String(l)).collect() +//! )), +//! _ => None, +//! } +//! } +//! } +//! +//! let expr = parse("state = \"open\"").unwrap().unwrap(); +//! let issue = Issue { state: "open".into(), priority: 1, labels: vec![] }; +//! assert!(expr.evaluate(&issue)); +//! ``` + +pub mod ast; +pub mod display; +pub mod eval; +pub mod lex; +pub mod parser; + +pub use ast::{BinaryOp, CompareOp, Expr, Position, UnaryOp}; +pub use eval::{Filterable, Value}; + +/// Parse a filter expression string into an [`Expr`] AST. +/// +/// Returns `Ok(None)` for empty/whitespace-only input. +/// Returns `Ok(Some(expr))` on success. +/// Returns `Err(message)` on parse errors. +pub fn parse(input: &str) -> Result<Option<Expr>, String> { + parser::parse_filter(input) +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..5bd138a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,155 @@ +use aip_filter::{parse, Expr, Filterable, Value}; +use serde_json::Value as Json; +use std::{ + fs::File, + io::{self, BufRead, BufReader, BufWriter, Write}, + path::Path, + process, +}; + +struct JsonRecord<'j>(&'j Json); + +impl<'j> Filterable for JsonRecord<'j> { + fn field(&self, name: &str) -> Option<Value<'_>> { + json_to_value(self.0.get(name)?) + } + + fn field_path(&self, path: &[&str]) -> Option<Value<'_>> { + let mut cur = self.0; + for &segment in path { + cur = cur.get(segment)?; + } + json_to_value(cur) + } + + fn all_field_values(&self) -> Vec<Value<'_>> { + match self.0 { + Json::Object(map) => map.values().filter_map(json_to_value).collect(), + _ => vec![], + } + } +} + +fn json_to_value(v: &Json) -> Option<Value<'_>> { + match v { + Json::String(s) => Some(Value::String(s.as_str())), + Json::Bool(b) => Some(Value::Bool(*b)), + Json::Null => Some(Value::Null), + Json::Number(n) => { + if let Some(i) = n.as_i64() { return Some(Value::Int(i)); } + if let Some(u) = n.as_u64() { return Some(Value::Uint(u)); } + n.as_f64().map(Value::Float) + } + Json::Array(arr) => Some(Value::List( + arr.iter().filter_map(json_to_value).collect(), + )), + // Nested objects are accessible via field_path; skip at top level. + Json::Object(_) => None, + } +} + +fn filter_lines( + expr: &Expr, + mut reader: impl BufRead, + out: &mut impl Write, + source: &str, +) -> io::Result<()> { + let mut line_buffer = String::with_capacity(128); + let mut lineno = 0; + + while reader.read_line(&mut line_buffer)? > 0 { + lineno += 1; + + // Skip empty lines + if !line_buffer.trim().is_empty() { + match serde_json::from_str::<Json>(&line_buffer) { + Ok(json) => { + if expr.evaluate(&JsonRecord(&json)) { + // Write directly to stdout + out.write_all(line_buffer.as_bytes())?; + } + } + Err(e) => { + eprintln!("{}:{}: invalid JSON: {}", source, lineno, e); + } + } + } + + // Clear the buffer for the next line, + // which keeps the memory allocated but resets the length to 0. + line_buffer.clear(); + } + + Ok(()) +} + +fn usage(program: &str) -> ! { + eprintln!("usage: {program} [-p|--print] <expr> [file...]"); + process::exit(2); +} + +fn main() { + let mut args: Vec<String> = std::env::args().collect(); + let program = args.remove(0); + + if args.is_empty() || args[0] == "-h" || args[0] == "--help" { + usage(&program); + } + + // Check for print flag + let mut print_only = false; + if args[0] == "-p" || args[0] == "--print" { + print_only = true; + args.remove(0); + if args.is_empty() { usage(&program); } + } + + let raw_expr = args.remove(0); + let expr = match parse(&raw_expr) { + Ok(Some(e)) => e, + Ok(None) => { + // Empty expression matches everything; just cat the input. + let stdin = io::stdin(); + let mut out = BufWriter::new(io::stdout().lock()); + for line in stdin.lock().lines() { + writeln!(out, "{}", line.unwrap_or_default()).ok(); + } + return; + } + Err(e) => { + eprintln!("{program}: {e}"); + process::exit(1); + } + }; + + if print_only { + println!("{}", expr); + return; + } + + let mut out = BufWriter::new(io::stdout().lock()); + + if args.is_empty() { + // Read from stdin. + let stdin = io::stdin(); + if let Err(e) = filter_lines(&expr, stdin.lock(), &mut out, "<stdin>") { + eprintln!("{program}: {e}"); + process::exit(1); + } + } else { + for path in &args { + let source = path.as_str(); + let file = match File::open(Path::new(path)) { + Ok(f) => f, + Err(e) => { + eprintln!("{program}: {path}: {e}"); + process::exit(1); + } + }; + if let Err(e) = filter_lines(&expr, BufReader::new(file), &mut out, source) { + eprintln!("{program}: {path}: {e}"); + process::exit(1); + } + } + } +} diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..7555a87 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,546 @@ +use super::ast::{BinaryOp, CompareOp, Expr, Position, UnaryOp}; +use super::lex::{Lexer, Token, TokenKind}; + +/// Parse a filter expression string into an [`Expr`]. +/// +/// Returns `Ok(None)` for empty input, `Ok(Some(expr))` on success, +/// `Err(msg)` on parse error. +pub fn parse_filter(filter: &str) -> Result<Option<Expr>, String> { + let mut lex = Lexer::new(filter); + + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + if tok.kind == TokenKind::Eof { + return Ok(None); + } + lex.push_token(tok); + + let expr = parse_expression(&mut lex)?; + + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + if tok.kind != TokenKind::Eof { + return Err(format!("{}: unexpected tokens after filter expression", tok.pos)); + } + + Ok(Some(expr)) +} + +/// expression = sequence, { "AND", sequence } ; +fn parse_expression(lex: &mut Lexer) -> Result<Expr, String> { + parse_junction(lex, TokenKind::And, parse_sequence) +} + +/// sequence = factor, { factor } ; +fn parse_sequence(lex: &mut Lexer) -> Result<Expr, String> { + let mut factor = parse_factor(lex)?; + + loop { + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + lex.push_token(tok.clone()); + + if tok.kind == TokenKind::Eof || tok.kind == TokenKind::Rparen { + return Ok(factor); + } + + let pos = tok.pos; + + if tok.kind == TokenKind::And { + return Ok(factor); + } + + let rfactor = parse_factor(lex)?; + + factor = Expr::Binary { + op: BinaryOp::And, + left: Box::new(factor), + right: Box::new(rfactor), + pos, + }; + } +} + +/// factor = term, { "OR", term } ; +fn parse_factor(lex: &mut Lexer) -> Result<Expr, String> { + parse_junction(lex, TokenKind::Or, parse_term) +} + +/// Parse an AND or OR sequence. +fn parse_junction( + lex: &mut Lexer, + kind: TokenKind, + parse_element: fn(&mut Lexer) -> Result<Expr, String>, +) -> Result<Expr, String> { + let mut sub = parse_element(lex)?; + + loop { + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + + if tok.kind != kind { + lex.push_token(tok); + return Ok(sub); + } + + let pos = tok.pos; + + tok = lex.next_token(); + if tok.kind != TokenKind::Wsc { + lex.push_token(tok); + let name = if kind == TokenKind::And { "AND" } else { "OR" }; + return Err(format!("{}: missing whitespace after {}", pos, name)); + } + + let rsub = parse_element(lex)?; + + let op = if kind == TokenKind::And { + BinaryOp::And + } else { + BinaryOp::Or + }; + sub = Expr::Binary { + op, + left: Box::new(sub), + right: Box::new(rsub), + pos, + }; + } +} + +/// term = [ "-" | "NOT" ], primitive ; +fn parse_term(lex: &mut Lexer) -> Result<Expr, String> { + let tok = lex.next_token(); + let is_neg = tok.kind == TokenKind::Minus || tok.kind == TokenKind::Not; + let neg_pos = tok.pos; + let neg_kind = tok.kind; + + if !is_neg { + lex.push_token(tok); + } + + if neg_kind == TokenKind::Not { + let stok = lex.next_token(); + if stok.kind != TokenKind::Wsc { + lex.push_token(stok); + return Err(format!("{}: missing whitespace after NOT", neg_pos)); + } + } + + let ntok = lex.next_token(); + lex.push_token(ntok.clone()); + let prim = if ntok.kind == TokenKind::Minus || ntok.kind == TokenKind::Not { + parse_term(lex)? + } else { + parse_primitive(lex)? + }; + + if is_neg { + let op = if neg_kind == TokenKind::Minus { + UnaryOp::Minus + } else { + UnaryOp::Not + }; + Ok(Expr::Unary { + op, + expr: Box::new(prim), + pos: neg_pos, + }) + } else { + Ok(prim) + } +} + +/// Parse a primitive (comparison, function call, parenthesized expr, or bare value). +fn parse_primitive(lex: &mut Lexer) -> Result<Expr, String> { + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + + if tok.kind == TokenKind::Lparen { + return parse_parenthesized_expression(lex); + } + + let left = parse_comparable(lex, tok)?; + + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + + let cmp_op = match tok.kind { + TokenKind::LessThanEquals => Some(CompareOp::LessThanEquals), + TokenKind::LessThan => Some(CompareOp::LessThan), + TokenKind::GreaterThanEquals => Some(CompareOp::GreaterThanEquals), + TokenKind::GreaterThan => Some(CompareOp::GreaterThan), + TokenKind::NotEquals => Some(CompareOp::NotEquals), + TokenKind::Equals => Some(CompareOp::Equals), + TokenKind::Has => Some(CompareOp::Has), + TokenKind::MatchesRegexp => Some(CompareOp::MatchesRegexp), + TokenKind::NotMatchesRegexp => Some(CompareOp::NotMatchesRegexp), + _ => None, + }; + + if let Some(op) = cmp_op { + let pos = tok.pos; + + let ntok = lex.next_token(); + if ntok.kind != TokenKind::Wsc { + lex.push_token(ntok); + } + + let right = parse_arg(lex)?; + + // Validate regex on the right-hand side + if op == CompareOp::MatchesRegexp || op == CompareOp::NotMatchesRegexp { + validate_regex(&right)?; + } + + Ok(Expr::Comparison { + op, + left: Box::new(left), + right: Box::new(right), + pos, + }) + } else { + lex.push_token(tok); + Ok(left) + } +} + +/// Validate that a regex RHS only contains quoted strings. +fn validate_regex(expr: &Expr) -> Result<(), String> { + match expr { + Expr::Binary { left, right, .. } => { + validate_regex(left)?; + validate_regex(right) + } + Expr::Unary { expr, .. } => validate_regex(expr), + Expr::Name { is_string, pos, name, .. } => { + if !is_string { + return Err(format!("{}: regular expression is not a quoted string", pos)); + } + // Validate the regex syntax + if let Err(e) = regex::Regex::new(name) { + return Err(format!("{}: invalid regular expression: {}", pos, e)); + } + Ok(()) + } + Expr::Comparison { pos, .. } | Expr::Member { pos, .. } => { + Err(format!("{}: regular expression is not a quoted string", pos)) + } + Expr::Function { func, .. } => { + // Use the function name position + let pos = func_pos(func); + Err(format!("{}: regular expression is not a quoted string", pos)) + } + } +} + +fn func_pos(expr: &Expr) -> Position { + match expr { + Expr::Name { pos, .. } | Expr::Member { pos, .. } => *pos, + _ => Position { line: 0, col: 0 }, + } +} + +/// Parse a parenthesized expression. We have already consumed the `(`. +fn parse_parenthesized_expression(lex: &mut Lexer) -> Result<Expr, String> { + let tok = lex.next_token(); + if tok.kind != TokenKind::Wsc { + lex.push_token(tok); + } + + let mut ret = parse_expression(lex)?; + + // Mark all name exprs as composite + mark_composite(&mut ret); + + let tok = lex.next_token(); + if tok.kind != TokenKind::Rparen { + lex.push_token(tok.clone()); + return Err(format!( + "{}: expected right parenthesis after expression", + tok.pos + )); + } + + Ok(ret) +} + +fn mark_composite(expr: &mut Expr) { + match expr { + Expr::Binary { left, right, .. } => { + mark_composite(left); + mark_composite(right); + } + Expr::Unary { expr, .. } => { + mark_composite(expr); + } + Expr::Comparison { left, right, .. } => { + mark_composite(left); + mark_composite(right); + } + Expr::Name { is_composite, .. } => { + *is_composite = true; + } + Expr::Member { holder, .. } => { + mark_composite(holder); + } + Expr::Function { .. } => {} + } +} + +/// Parse a comparable (name, member, function). +fn parse_comparable(lex: &mut Lexer, tok: Token) -> Result<Expr, String> { + let mut saw_white = false; + let mut ntok = lex.next_token(); + if ntok.kind == TokenKind::Wsc { + saw_white = true; + ntok = lex.next_token(); + } + lex.push_token(ntok.clone()); + + let tok = match tok.kind { + TokenKind::StringLit | TokenKind::Text => tok, + TokenKind::And | TokenKind::Or | TokenKind::Not => { + if ntok.kind != TokenKind::Dot { + return Err(format!("{}: misplaced AND/OR/NOT", tok.pos)); + } + keyword_to_text(tok) + } + _ => return Err(format!("{}: expected identifier or value", tok.pos)), + }; + + match ntok.kind { + TokenKind::Dot => { + lex.next_token(); // consume dot + let dot_pos = ntok.pos; + let e = parse_member(lex, &tok, dot_pos)?; + + // Check if member is used as a function call + let ftok = lex.next_token(); + if ftok.kind != TokenKind::Lparen || saw_white { + lex.push_token(ftok); + return Ok(e); + } + parse_function(lex, e) + } + TokenKind::Lparen => { + let fn_expr = Expr::Name { + name: tok.val.clone(), + pos: tok.pos, + is_string: false, + is_composite: false, + }; + + if saw_white || tok.val.is_empty() || tok.val.starts_with(|c: char| c.is_ascii_digit()) + { + return Ok(fn_expr); + } + + lex.next_token(); // consume lparen + parse_function(lex, fn_expr) + } + _ => Ok(Expr::Name { + name: tok.val, + pos: tok.pos, + is_string: tok.kind == TokenKind::StringLit, + is_composite: false, + }), + } +} + +/// Parse member access: `identifier { "." identifier }`. +fn parse_member(lex: &mut Lexer, tok: &Token, pos: Position) -> Result<Expr, String> { + let mut ret: Expr = Expr::Name { + name: tok.val.clone(), + pos: tok.pos, + is_string: false, + is_composite: false, + }; + + let mut dot_pos = pos; + + loop { + let ntok = lex.next_token(); + let ntok = match ntok.kind { + TokenKind::StringLit | TokenKind::Text => ntok, + TokenKind::And | TokenKind::Or | TokenKind::Not => keyword_to_text(ntok), + _ => return Err(format!("{}: expected identifier", ntok.pos)), + }; + + ret = Expr::Member { + holder: Box::new(ret), + member: ntok.val, + pos: dot_pos, + }; + + let next = lex.next_token(); + if next.kind != TokenKind::Dot { + lex.push_token(next); + return Ok(ret); + } + dot_pos = next.pos; + } +} + +/// Parse a function call. We've consumed the `(`. +fn parse_function(lex: &mut Lexer, func: Expr) -> Result<Expr, String> { + let tok = lex.next_token(); + if tok.kind == TokenKind::Rparen { + return Ok(Expr::Function { + func: Box::new(func), + args: vec![], + }); + } + lex.push_token(tok); + + let mut args = Vec::new(); + loop { + let arg = parse_arg(lex)?; + args.push(arg); + + let mut tok = lex.next_token(); + if tok.kind == TokenKind::Rparen { + return Ok(Expr::Function { + func: Box::new(func), + args, + }); + } + if tok.kind == TokenKind::Wsc { + tok = lex.next_token(); + } + if tok.kind != TokenKind::Comma { + return Err(format!( + "{}: expected comma after function argument", + tok.pos + )); + } + + tok = lex.next_token(); + if tok.kind != TokenKind::Wsc { + lex.push_token(tok); + } + } +} + +/// Parse an argument (comparable, value, or parenthesized expression). +fn parse_arg(lex: &mut Lexer) -> Result<Expr, String> { + let tok = lex.next_token(); + if tok.kind == TokenKind::Lparen { + return parse_parenthesized_expression(lex); + } + parse_comparable(lex, tok) +} + +/// Convert a keyword token (AND/OR/NOT) to a text token. +fn keyword_to_text(tok: Token) -> Token { + let val = match tok.kind { + TokenKind::And => "AND", + TokenKind::Or => "OR", + TokenKind::Not => "NOT", + _ => unreachable!(), + }; + Token { + kind: TokenKind::Text, + pos: tok.pos, + val: val.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_empty() { + assert!(parse_filter("").unwrap().is_none()); + assert!(parse_filter(" ").unwrap().is_none()); + } + + #[test] + fn test_basic_comparison() { + let expr = parse_filter("x = 42").unwrap().unwrap(); + match expr { + Expr::Comparison { op, left, right, .. } => { + assert_eq!(op, CompareOp::Equals); + match *left { + Expr::Name { ref name, .. } => assert_eq!(name, "x"), + _ => panic!("expected name"), + } + match *right { + Expr::Name { ref name, .. } => assert_eq!(name, "42"), + _ => panic!("expected name"), + } + } + _ => panic!("expected comparison"), + } + } + + #[test] + fn test_conjunction() { + let expr = parse_filter("a : b c = 17").unwrap().unwrap(); + match expr { + Expr::Binary { op: BinaryOp::And, .. } => {} + _ => panic!("expected conjunction"), + } + } + + #[test] + fn test_negation() { + let expr = parse_filter("-a < 1").unwrap().unwrap(); + match expr { + Expr::Unary { op: UnaryOp::Minus, .. } => {} + _ => panic!("expected unary minus"), + } + } + + #[test] + fn test_member() { + let expr = parse_filter("a.b > 17").unwrap().unwrap(); + match expr { + Expr::Comparison { left, .. } => match *left { + Expr::Member { ref member, .. } => assert_eq!(member, "b"), + _ => panic!("expected member"), + }, + _ => panic!("expected comparison"), + } + } + + #[test] + fn test_function() { + let expr = parse_filter("func(val)").unwrap().unwrap(); + match expr { + Expr::Function { func, args } => { + match *func { + Expr::Name { ref name, .. } => assert_eq!(name, "func"), + _ => panic!("expected name"), + } + assert_eq!(args.len(), 1); + } + _ => panic!("expected function"), + } + } + + #[test] + fn test_error_double_equals() { + let err = parse_filter("x == 42").unwrap_err(); + assert!(err.contains("expected identifier or value"), "got: {err}"); + } + + #[test] + fn test_error_misplaced_keyword() { + let err = parse_filter("a : AND b c = 17").unwrap_err(); + assert!(err.contains("misplaced AND/OR/NOT"), "got: {err}"); + } +} diff --git a/testdata/parse/basic.txt b/testdata/parse/basic.txt new file mode 100644 index 0000000..bc16e53 --- /dev/null +++ b/testdata/parse/basic.txt @@ -0,0 +1,29 @@ +# Basic parsing tests. +-- basic1.test -- +x = 42 +-- basic1.out -- +compare = + x + 42 +-- basic2.test -- +a : b c = 17 +-- basic2.out -- +conjunction + compare : + a + b + compare = + c + 17 +-- basic3.test -- +x == 42 +-- basic3.err -- +1:3: expected identifier or value +-- basic4.test -- +a : AND b c = 17 +-- basic4.err -- +1:4: misplaced AND/OR/NOT +-- basic5.test -- +a : b c = 17 - +-- basic5.err -- +2:0: expected identifier or value diff --git a/testdata/parse/basic2.txt b/testdata/parse/basic2.txt new file mode 100644 index 0000000..5cc996f --- /dev/null +++ b/testdata/parse/basic2.txt @@ -0,0 +1,69 @@ +# More basic parsing tests. +-- basic10.test -- +foo +-- basic10.out -- +foo +-- basic11.test -- +foo() +-- basic11.out -- +call foo +-- basic12.test -- +foo = "bar" +-- basic12.out -- +compare = + foo + "bar" +-- basic13.test -- +foo=42s +-- basic13.out -- +compare = + foo + 42s +-- basic14.test -- +foo=4.2.s +-- basic14.out -- +compare = + foo + 4.2.s +-- basic15.test -- +foo=42.1s +-- basic15.out -- +compare = + foo + 42.1s +-- basic16.test -- +foo=42.1337s +-- basic16.out -- +compare = + foo + 42.1337s +-- basic17.test -- +foo=42.1337.24601s +-- basic17.out -- +compare = + foo + 42.1337.24601s +-- basic18.test -- +foo=42.s +-- basic18.out -- +compare = + foo + 42.s +-- basic19.test -- +foo=.42.s +-- basic19.out -- +compare = + foo + .42.s +-- basic20.test -- +foo..1.2 +-- basic20.out -- +foo..1.2 +-- basic21.test -- +foo..1..2 +-- basic21.out -- +foo..1..2 +-- basic22.test -- +2019-01..2023-01 +-- basic22.out -- +2019-01..2023-01 diff --git a/testdata/parse/comments.txt b/testdata/parse/comments.txt new file mode 100644 index 0000000..0bd3e04 --- /dev/null +++ b/testdata/parse/comments.txt @@ -0,0 +1,108 @@ +# Comment tests. +-- comment1.test -- +-- +-- comment1.out -- +-- comment2.test -- +-- Just a comment single line +-- comment2.out -- +-- comment3.test -- + -- Just an indented comment single line +-- comment3.out -- +-- comment4.test -- + + -- Just a comment + -- + -- a lone comment + +-- comment4.out -- +-- comment5.test -- + + term + -- Ending comment + +-- comment5.out -- +term +-- comment6.test -- + + -- opening comment + term + +-- comment6.out -- +term +-- comment7.test -- + + term -- same-line comment + +-- comment7.out -- +term +-- comment8.test -- + + foo + -- implicit conjunction preceding comment + bar + baz + +-- comment8.out -- +conjunction + conjunction + foo + bar + baz +-- comment9.test -- + + foo + bar -- implicit conjunction inline comment + baz + +-- comment9.out -- +conjunction + conjunction + foo + bar + baz +-- comment10.test -- + + foo AND + bar AND -- explicit conjunction inline comment + baz + +-- comment10.out -- +conjunction + conjunction + foo + bar + baz +-- comment11.test -- + + foo OR + bar OR -- explicit disjunction inline comment + baz + +-- comment11.out -- +disjunction + disjunction + foo + bar + baz +-- comment12.test -- + + -- multi-line + -- comment + -- block + (foo OR -- next one + bar) + bar + --comment 1 + baz + qux -- comment 2 + -- comment 3 +-- comment12.out -- +conjunction + conjunction + conjunction + disjunction + foo + bar + bar + baz + qux diff --git a/testdata/parse/conjunction.txt b/testdata/parse/conjunction.txt new file mode 100644 index 0000000..df6252f --- /dev/null +++ b/testdata/parse/conjunction.txt @@ -0,0 +1,21 @@ +# Conjunction tests. +-- conjunction1.test -- + a : b c = 17 +-- conjunction1.out -- +conjunction + compare : + a + b + compare = + c + 17 +-- conjunction2.test -- +date>yesterday AND size<20 +-- conjunction2.out -- +conjunction + compare > + date + yesterday + compare < + size + 20 diff --git a/testdata/parse/disjunction.txt b/testdata/parse/disjunction.txt new file mode 100644 index 0000000..b22efae --- /dev/null +++ b/testdata/parse/disjunction.txt @@ -0,0 +1,11 @@ +# Disjunction tests. +-- disjunction1.test -- +a:1 OR b=2 +-- disjunction1.out -- +disjunction + compare : + a + 1 + compare = + b + 2 diff --git a/testdata/parse/empty.txt b/testdata/parse/empty.txt new file mode 100644 index 0000000..be4be4b --- /dev/null +++ b/testdata/parse/empty.txt @@ -0,0 +1,12 @@ +# Empty input tests. +-- empty1.test -- +-- empty1.out -- +-- empty2.test -- + +-- empty2.out -- +-- empty3.test -- +route = /a/route* +-- empty3.out -- +compare = + route + /a/route* diff --git a/testdata/parse/equality.txt b/testdata/parse/equality.txt new file mode 100644 index 0000000..bb11f80 --- /dev/null +++ b/testdata/parse/equality.txt @@ -0,0 +1,43 @@ +# Equality tests. +-- equality1.test -- +a!=b +-- equality1.out -- +compare != + a + b +-- equality2.test -- +a != b +-- equality2.out -- +compare != + a + b +-- equality3.test -- +a!= b +-- equality3.out -- +compare != + a + b +-- equality4.test -- +a !=b +-- equality4.out -- +compare != + a + b +-- equality5.test -- +a!b!=c +-- equality5.out -- +compare != + a!b + c +-- equality6.test -- +a!=b!c +-- equality6.out -- +compare != + a + b!c +-- equality7.test -- +a=/b/ +-- equality7.out -- +compare = + a + /b/ diff --git a/testdata/parse/error.txt b/testdata/parse/error.txt new file mode 100644 index 0000000..efbfe6a --- /dev/null +++ b/testdata/parse/error.txt @@ -0,0 +1,38 @@ +# Erroneous tests +-- error1.test -- +x == 42 +-- error1.err -- +1:3: expected identifier or value +-- error2.test -- +"\+" +-- error2.out -- +"+" +-- error3.test -- +"\*" +-- error3.out -- +"*" +-- error4.test -- +"\>" +-- error4.out -- +">" +-- error5.test -- +"YěHN摁Băo\+钬徯h?+8刄" +-- error5.out -- +"YěHN摁Băo+钬徯h?+8刄" +-- error6.test -- +-"ŊWf鿅qĒĘSɇ息¢f\*" +-- error6.out -- +minus + "ŊWf鿅qĒĘSɇ息¢f*" +-- error7.test -- +"炠Ě戌N*劥y畝\+" +-- error7.out -- +"炠Ě戌N*劥y畝+" +-- error8.test -- +"缻\\>Ȥ恶´帛砜Ɏ拟" +-- error8.out -- +"缻\\>Ȥ恶´帛砜Ɏ拟" +-- error9.test -- +"\xa0\xa1" +-- error9.out -- +"\u{a0}¡" diff --git a/testdata/parse/error2.txt b/testdata/parse/error2.txt new file mode 100644 index 0000000..2c465f3 --- /dev/null +++ b/testdata/parse/error2.txt @@ -0,0 +1,129 @@ +# More parser error tests. +-- error10.test -- +a: .a +-- error10.err -- +1:3: expected identifier or value +-- error11.test -- +a:-b +-- error11.err -- +1:2: expected identifier or value +-- error12.test -- +a[$b] ~ "c*" +-- error12.err -- +1:6: expected identifier or value +-- error13.test -- +( +-- error13.err -- +2:0: expected identifier or value +-- error14.test -- +argument(1, 2 +-- error14.err -- +2:0: expected comma after function argument +-- error15.test -- +.( +-- error15.err -- +1:0: expected identifier or value +-- error16.test -- +- +-- error16.err -- +2:0: expected identifier or value +-- error17.test -- +=~ +-- error17.err -- +1:0: expected identifier or value +-- error18.test -- +!~ +-- error18.err -- +1:0: expected identifier or value +-- error19.test -- +NOT +-- error19.err -- +2:0: expected identifier or value +-- error20.test -- +" +-- error20.err -- +1:1: expected identifier or value +-- error21.test -- +a b +-- error21.err -- +1:2: expected identifier or value +-- error22.test -- +() +-- error22.err -- +1:1: expected identifier or value +-- error23.test -- +\u10 +-- error23.err -- +1:1: expected identifier or value +-- error24.test -- +a: +-- error24.err -- +2:0: expected identifier or value +-- error25.test -- +=123 +-- error25.err -- +1:0: expected identifier or value +-- error26.test -- +(foo OR (bar baz) +-- error26.err -- +2:0: expected right parenthesis after expression +-- error27.test -- +foo OR AND bar +-- error27.err -- +1:7: misplaced AND/OR/NOT +-- error28.test -- +foo, bar +-- error28.err -- +1:3: expected identifier or value +-- error29.test -- +foo=+-3 +-- error29.err -- +1:4: expected identifier or value +-- error30.test -- +foo>>3 +-- error30.err -- +1:4: expected identifier or value +-- error31.test -- +foo..bar=3 +-- error31.err -- +1:4: expected identifier +-- error32.test -- +foo."bar=3 +-- error32.err -- +1:5: expected identifier +-- error33.test -- +"abc \xa0 xyz" +-- error33.out -- +"abc \u{a0} xyz" +-- error34.test -- +foo = "abc \xa0 xyz" +-- error34.out -- +compare = + foo + "abc \u{a0} xyz" +-- error35.test -- +"abc \xa0 xyz" +-- error35.out -- +"abc \u{a0} xyz" +-- error36.test -- +foo =~ /no-terminating-slash +-- error36.err -- +1:8: regular expression is not a quoted string +-- error37.test -- +AND +-- error37.err -- +1:0: misplaced AND/OR/NOT +-- error38.test -- +:hello +-- error38.err -- +1:0: expected identifier or value +-- error39.test -- +hello: +-- error39.err -- +2:0: expected identifier or value +-- error40.test -- + + +hello: +-- error40.err -- +4:0: expected identifier or value diff --git a/testdata/parse/expr.txt b/testdata/parse/expr.txt new file mode 100644 index 0000000..eb87924 --- /dev/null +++ b/testdata/parse/expr.txt @@ -0,0 +1,21 @@ +# Expression tests +-- expr1.test -- +abc +-- expr1.out -- +abc +-- expr2.test -- +\+YěHN摁Băo +-- expr2.out -- ++YěHN摁Băo +-- expr3.test -- +ŊWf鿅qĒĘSɇ息¢f +-- expr3.out -- +ŊWf鿅qĒĘSɇ息¢f +-- expr4.test -- +炠Ě戌N*劥y畝 +-- expr4.out -- +炠Ě戌N*劥y畝 +-- expr5.test -- +缻\>Ȥ恶´帛砜Ɏ拟 +-- expr5.out -- +缻>Ȥ恶´帛砜Ɏ拟 diff --git a/testdata/parse/function.txt b/testdata/parse/function.txt new file mode 100644 index 0000000..1a20c82 --- /dev/null +++ b/testdata/parse/function.txt @@ -0,0 +1,36 @@ +# Function tests. +-- func1.test -- +func(val) +-- func1.out -- +call func + val +-- func2.test -- +a: func(val) +-- func2.out -- +compare : + a + call func + val +-- func3.test -- +n1.n2.n3.func(val) +-- func3.out -- +call n1.n2.n3.func + val +-- func4.test -- +a: n1.n2.func(val) +-- func4.out -- +compare : + a + call n1.n2.func + val +-- func5.test -- +a.b() +-- func5.out -- +call a.b +-- func6.test -- +x > a.b.c(d) +-- func6.out -- +compare > + x + call a.b.c + d diff --git a/testdata/parse/function2.txt b/testdata/parse/function2.txt new file mode 100644 index 0000000..b24e703 --- /dev/null +++ b/testdata/parse/function2.txt @@ -0,0 +1,41 @@ +# More function tests. +-- func10.test -- +!@#$%^&*() +-- func10.out -- +call !@#$%^&* +-- func11.test -- +!@#$%^&*(arg) +-- func11.out -- +call !@#$%^&* + arg +-- func12.test -- +foo() +-- func12.out -- +call foo +-- func13.test -- +a > foo() +-- func13.out -- +compare > + a + call foo +-- func14.test -- +b < foo(3) +-- func14.out -- +compare < + b + call foo + 3 +-- func15.test -- +a > b.c.d(e) +-- func15.out -- +compare > + a + call b.c.d + e +-- func16.test -- +a > b.c.d(e) +-- func16.out -- +compare > + a + call b.c.d + e diff --git a/testdata/parse/member.txt b/testdata/parse/member.txt new file mode 100644 index 0000000..23486a1 --- /dev/null +++ b/testdata/parse/member.txt @@ -0,0 +1,57 @@ +# Member tests. +-- member1.test -- +a.b > 17 +-- member1.out -- +compare > + a.b + 17 +-- member2.test -- +a."b" > 7.6 +-- member2.out -- +compare > + a.b + 7.6 +-- member3.test -- +a.3.4 > 5.6 +-- member3.out -- +compare > + a.3.4 + 5.6 +-- member4.test -- +a."3"."4" > 5.6 +-- member4.out -- +compare > + a.3.4 + 5.6 +-- member5.test -- +a.b.c.d > 17 +-- member5.out -- +compare > + a.b.c.d + 17 +-- member6.test -- +a.2.3.4.5 > 3.14 +-- member6.out -- +compare > + a.2.3.4.5 + 3.14 +-- member7.test -- +1.2.3.4.5 > 3.14 +-- member7.out -- +compare > + 1.2.3.4.5 + 3.14 +-- member8.test -- +a.b +-- member8.out -- +a.b +-- member9.test -- +a[b] +-- member9.out -- +a[b] +-- member10.test -- +a.NOT.OR.AND:value +-- member10.out -- +compare : + a.NOT.OR.AND + value diff --git a/testdata/parse/negation.txt b/testdata/parse/negation.txt new file mode 100644 index 0000000..588ca6e --- /dev/null +++ b/testdata/parse/negation.txt @@ -0,0 +1,33 @@ +# Negation tests. +-- negation1.test -- +-a < 1 +-- negation1.out -- +minus + compare < + a + 1 +-- negation2.test -- +NOT a > 3 +-- negation2.out -- +not + compare > + a + 3 +-- negation3.test -- +-foo=(bar OR baz) +-- negation3.out -- +minus + compare = + foo + disjunction + bar + baz +-- negation4.test -- +NOT foo=(bar OR baz) +-- negation4.out -- +not + compare = + foo + disjunction + bar + baz diff --git a/testdata/parse/other.txt b/testdata/parse/other.txt new file mode 100644 index 0000000..abacfba --- /dev/null +++ b/testdata/parse/other.txt @@ -0,0 +1,64 @@ +# Other random tests. +-- other1.test -- +a[ argument(1) ]: latlng(3.0, 5.3) +-- other1.out -- +conjunction + conjunction + a[ + call argument + 1 + compare : + ] + call latlng + 3.0 + 5.3 +-- other2.test -- +a[ b ] >= 7.6 +-- other2.out -- +conjunction + conjunction + a[ + b + compare >= + ] + 7.6 +-- other3.test -- +a[ "b" ] <= 3.14 +-- other3.out -- +conjunction + conjunction + a[ + "b" + compare <= + ] + 3.14 +-- other4.test -- +field : parent.child +-- other4.out -- +compare : + field + parent.child +-- other5.test -- +field : foo* +-- other5.out -- +compare : + field + foo* +-- other6.test -- +field : foo\* +-- other6.out -- +compare : + field + foo* +-- other7.test -- +/foo.bar/ +-- other7.out -- +/foo.bar/ +-- other8.test -- +((foo)) +-- other8.out -- +foo +-- other9.test -- +((foo)) +-- other9.out -- +foo diff --git a/testdata/parse/parentheses.txt b/testdata/parse/parentheses.txt new file mode 100644 index 0000000..0b6ea0e --- /dev/null +++ b/testdata/parse/parentheses.txt @@ -0,0 +1,19 @@ +# Tests with parentheses +-- parentheses1.test -- +(a>1) OR ((b<2 AND c:3) AND d=4) +-- parentheses1.out -- +disjunction + compare > + a + 1 + conjunction + conjunction + compare < + b + 2 + compare : + c + 3 + compare = + d + 4 diff --git a/testdata/parse/phrase.txt b/testdata/parse/phrase.txt new file mode 100644 index 0000000..4bae3f7 --- /dev/null +++ b/testdata/parse/phrase.txt @@ -0,0 +1,99 @@ +# Phrase tests. +-- phrase1.test -- +empty:"" +-- phrase1.out -- +compare : + empty + "" +-- phrase2.test -- +non-empty:"phrase" +-- phrase2.out -- +compare : + non-empty + "phrase" +-- phrase3.test -- +unicode:"Да!" +-- phrase3.out -- +compare : + unicode + "Да!" +-- phrase4.test -- +value:"\x66\x6f\x6F" +-- phrase4.out -- +compare : + value + "foo" +-- phrase5.test -- +X X +X XX X +X +-- phrase5.out -- +conjunction + conjunction + conjunction + conjunction + conjunction + conjunction + X + X + X + X + X + X + X +-- phrase6.test -- +payload:"not --a --comment=10 -- foo" +-- phrase6.out -- +compare : + payload + "not --a --comment=10 -- foo" +-- phrase7.test -- +value: "this is a phrase" +-- phrase7.out -- +compare : + value + "this is a phrase" +-- phrase8.test -- +value: "this is a phrase" +-- phrase8.out -- +compare : + value + "this is a phrase" +-- phrase9.test -- +value: "open quote +-- phrase9.err -- +1:8: expected identifier or value +-- phrase10.test -- +value: "open quote +-- phrase10.err -- +1:8: expected identifier or value +-- phrase11.test -- +value: "\, \: \= \< \> \+ \~ \" \\ \. \*" +-- phrase11.out -- +compare : + value + ", : = < > + ~ \" \\ . *" +-- phrase12.test -- +value: "\a \b \f \n \r \t \v" +-- phrase12.out -- +compare : + value + "\u{7} \u{8} \u{c} \n \r \t \u{b}" +-- phrase13.test -- +value:"(foo)-.bar" +-- phrase13.out -- +compare : + value + "(foo)-.bar" +-- phrase14.test -- + url = "https\://www\.google\.com/test\?a\=b" +-- phrase14.out -- +compare = + url + "https://www.google.com/test\\?a=b" +-- phrase15.test -- + url = "https\://www\.google\.com/test\?a\=b" +-- phrase15.out -- +compare = + url + "https://www.google.com/test\\?a=b" diff --git a/testdata/parse/prec.txt b/testdata/parse/prec.txt new file mode 100644 index 0000000..bcbab3f --- /dev/null +++ b/testdata/parse/prec.txt @@ -0,0 +1,9 @@ +# Precedence tests +-- prec1.test -- +a AND b OR c +-- prec1.out -- +conjunction + a + disjunction + b + c diff --git a/testdata/parse/regexp.txt b/testdata/parse/regexp.txt new file mode 100644 index 0000000..e85ab2d --- /dev/null +++ b/testdata/parse/regexp.txt @@ -0,0 +1,137 @@ +# Regexp tests. +-- regexp1.test -- +a=~"b" +-- regexp1.out -- +compare =~ + a + "b" +-- regexp2.test -- +a.b.c=~"b.*c" +-- regexp2.out -- +compare =~ + a.b.c + "b.*c" +-- regexp3.test -- +a.b.c=~"12[a-zA-Z]+x.*(a|b|c)?" +-- regexp3.out -- +compare =~ + a.b.c + "12[a-zA-Z]+x.*(a|b|c)?" +-- regexp4.test -- +a AND b=~"foo.*bar?" AND c AND d.e =~ "baz[^\"]+" +-- regexp4.out -- +conjunction + conjunction + conjunction + a + compare =~ + b + "foo.*bar?" + c + compare =~ + d.e + "baz[^\"]+" +-- regexp5.test -- +a =~ ("foo.*bar" AND ("foo.*baz" OR "foo.*qux")) +-- regexp5.out -- +compare =~ + a + conjunction + "foo.*bar" + disjunction + "foo.*baz" + "foo.*qux" +-- regexp6.test -- +a =~ word +-- regexp6.err -- +1:6: regular expression is not a quoted string +-- regexp7.test -- +a =~ ("foo.*" OR bar) +-- regexp7.err -- +1:18: regular expression is not a quoted string +-- regexp8.test -- +a =~ "(no|closing|parens" +-- regexp8.err -- +1:6: invalid regular expression: error parsing regexp: missing closing ): `(no|closing|parens` +-- regexp9.test -- +a =~ "[no|closing|brace" +-- regexp9.err -- +1:6: invalid regular expression: error parsing regexp: missing closing ]: `[no|closing|brace` +-- regexp10.test -- +a!~"b" +-- regexp10.out -- +compare !~ + a + "b" +-- regexp11.test -- +a.b.c!~"b.*c" +-- regexp11.out -- +compare !~ + a.b.c + "b.*c" +-- regexp12.test -- +a.b.c!~"12[a-zA-Z]+x.*(a|b|c)?" +-- regexp12.out -- +compare !~ + a.b.c + "12[a-zA-Z]+x.*(a|b|c)?" +-- regexp13.test -- +a AND b!~"foo.*bar?" AND c AND d.e !~ "baz[^\"]+" +-- regexp13.out -- +conjunction + conjunction + conjunction + a + compare !~ + b + "foo.*bar?" + c + compare !~ + d.e + "baz[^\"]+" +-- regexp14.test -- +a !~ ("foo.*bar" AND ("foo.*baz" OR "foo.*qux")) +-- regexp14.out -- +compare !~ + a + conjunction + "foo.*bar" + disjunction + "foo.*baz" + "foo.*qux" +-- regexp15.test -- +a !~ word +-- regexp15.err -- +1:6: regular expression is not a quoted string +-- regexp16.test -- +a !~ ("foo.*" OR bar) +-- regexp16.err -- +1:18: regular expression is not a quoted string +-- regexp17.test -- +a !~ "(no|closing|parens" +-- regexp17.err -- +1:6: invalid regular expression: error parsing regexp: missing closing ): `(no|closing|parens` +-- regexp18.test -- +a !~ "[no|closing|brace" +-- regexp18.err -- +1:6: invalid regular expression: error parsing regexp: missing closing ]: `[no|closing|brace` +-- regexp19.test -- +a !~ "(no|closing|parens" +-- regexp19.err -- +1:6: invalid regular expression: error parsing regexp: missing closing ): `(no|closing|parens` +-- regexp20.test -- +a !~ "[no|closing|brace" +-- regexp20.err -- +1:6: invalid regular expression: error parsing regexp: missing closing ]: `[no|closing|brace` +-- regexp21.test -- +a=~"b" +-- regexp21.out -- +compare =~ + a + "b" +-- regexp22.test -- +a!~"b" +-- regexp22.out -- +compare !~ + a + "b" diff --git a/testdata/parse/unicode.txt b/testdata/parse/unicode.txt new file mode 100644 index 0000000..0020c72 --- /dev/null +++ b/testdata/parse/unicode.txt @@ -0,0 +1,59 @@ +# Unicode tests. +-- unicode1.test -- +fờö +-- unicode1.out -- +fờö +-- unicode2.test -- +answer:Да! +-- unicode2.out -- +compare : + answer + Да! +-- unicode3.test -- +metric.label.subject="ac�n" +-- unicode3.out -- +compare = + metric.label.subject + "ac�n" +-- unicode4.test -- +metric.label.subject="ac�n" +-- unicode4.out -- +compare = + metric.label.subject + "ac�n" +-- unicode5.test -- +你好吗 +-- unicode5.out -- +你好吗 +-- unicode6.test -- +"你好吗" +-- unicode6.out -- +"你好吗" +-- unicode7.test -- +"你好吗" +-- unicode7.out -- +"你好吗" +-- unicode8.test -- +🟦 +-- unicode8.out -- +🟦 +-- unicode9.test -- +"🟦" +-- unicode9.out -- +"🟦" +-- unicode10.test -- +"🟦" +-- unicode10.out -- +"🟦" +-- unicode11.test -- +"󠅢" +-- unicode11.out -- +"\u{e0162}" +-- unicode12.test -- +"\777" +-- unicode12.out -- +"?7" +-- unicode13.test -- +"\777" +-- unicode13.out -- +"?7" diff --git a/tests/parse_tests.rs b/tests/parse_tests.rs new file mode 100644 index 0000000..1f15bd2 --- /dev/null +++ b/tests/parse_tests.rs @@ -0,0 +1,153 @@ +use aip_filter::parse; +use std::fs; +use std::path::Path; + +/// Parse a txtar-style test file into named sections. +/// Sections are delimited by `-- name --` lines. +fn parse_txtar(content: &str) -> Vec<(String, String)> { + let mut files: Vec<(String, String)> = Vec::new(); + let mut current_name: Option<String> = None; + let mut current_data = String::new(); + + for line in content.split('\n') { + let trimmed = line.trim_end_matches('\r'); + if trimmed.starts_with("-- ") && trimmed.ends_with(" --") { + if let Some(name) = current_name.take() { + files.push((name, current_data.clone())); + } + let name = trimmed[3..trimmed.len() - 3].to_string(); + current_name = Some(name); + current_data.clear(); + } else if current_name.is_some() { + current_data.push_str(line); + current_data.push('\n'); + } + } + if let Some(name) = current_name { + files.push((name, current_data)); + } + files +} + +#[test] +fn test_parse() { + let test_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/parse"); + if !test_dir.exists() { + eprintln!("Skipping parse tests: testdata/parse not found"); + return; + } + + let mut total = 0; + let mut passed = 0; + let mut failed_tests = Vec::new(); + + for entry in fs::read_dir(&test_dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.extension().map(|e| e == "txt").unwrap_or(false) { + let content = fs::read_to_string(&path).unwrap(); + let files = parse_txtar(&content); + + let mut i = 0; + while i < files.len() { + let (name, data) = &files[i]; + if !name.ends_with(".test") { + i += 1; + continue; + } + + let base = name.strip_suffix(".test").unwrap(); + total += 1; + + let result = parse(data); + + i += 1; + if i >= files.len() { + eprintln!(" {}: missing expected result", base); + continue; + } + + let (expected_name, expected_data) = &files[i]; + + if expected_name.ends_with(".out") { + match result { + Err(e) => { + failed_tests.push(format!( + "{}:{}: got error '{}', expected success", + path.file_name().unwrap().to_str().unwrap(), + base, + e + )); + } + Ok(expr) => { + let got = match expr { + Some(e) => format!("{}", e), + None => String::new(), + }; + let want = expected_data.trim_end_matches('\n'); + let got_trimmed = got.trim_end_matches('\n'); + if got_trimmed == want { + passed += 1; + } else { + failed_tests.push(format!( + "{}:{}: output mismatch\n got: {:?}\n want: {:?}", + path.file_name().unwrap().to_str().unwrap(), + base, + got_trimmed, + want + )); + } + } + } + } else if expected_name.ends_with(".err") { + match result { + Ok(_) => { + failed_tests.push(format!( + "{}:{}: got success, expected error '{}'", + path.file_name().unwrap().to_str().unwrap(), + base, + expected_data.trim() + )); + } + Err(e) => { + let want = expected_data.trim(); + let got = e.trim(); + if got == want || regex_error_prefix_matches(got, want) { + passed += 1; + } else { + failed_tests.push(format!( + "{}:{}: error mismatch\n got: {:?}\n want: {:?}", + path.file_name().unwrap().to_str().unwrap(), + base, + got, + want + )); + } + } + } + } + } + } + } + + println!("\nParse tests: {}/{} passed", passed, total); + if !failed_tests.is_empty() { + println!("\nFailed tests:"); + for f in &failed_tests { + println!(" {}", f); + } + panic!("{} of {} parse tests failed", failed_tests.len(), total); + } +} + +/// Returns true when the position prefix and "invalid regular expression:" label +/// match but the trailing error detail does not - the regex crate's error messages +/// are more verbose than the testdata expects, so we accept any error at the right +/// position with the right label. +fn regex_error_prefix_matches(got: &str, want: &str) -> bool { + const MARKER: &str = "invalid regular expression:"; + match (got.split_once(MARKER), want.split_once(MARKER)) { + (Some((gp, _)), Some((wp, _))) => gp == wp, + _ => false, + } +} |
