vrl/datadog/search/
parser.rs

1use std::str::FromStr;
2
3use pest::Parser;
4
5use super::{
6    grammar::{DEFAULT_FIELD, EventPlatformQuery, QueryVisitor},
7    node::QueryNode,
8};
9
10pub type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
11
12/// Quick wrapper parse function to convert query strings into our AST
13impl FromStr for QueryNode {
14    type Err = Error;
15
16    fn from_str(query: &str) -> Result<Self, Self::Err> {
17        // Clean up our query string
18        let clean_query = query.trim();
19        // If we have an empty query, we presume we're matching everything
20        Ok(if clean_query.is_empty() {
21            Self::MatchAllDocs
22        } else {
23            // Otherwise parse and interpret the query
24            let mut ast = EventPlatformQuery::parse(super::grammar::Rule::queryroot, query)?;
25            let rootquery = ast.next().ok_or("Unable to find root query")?;
26            QueryVisitor::visit_queryroot(rootquery, DEFAULT_FIELD)
27        })
28    }
29}
30
31#[cfg(test)]
32mod tests {
33    use super::super::node::{BooleanType, Comparison, ComparisonValue, QueryNode};
34    use super::*;
35
36    fn parse(s: &str) -> QueryNode {
37        s.parse()
38            .unwrap_or_else(|error| panic!("Unable to parse {s:?}: {error}."))
39    }
40
41    #[test]
42    fn parses_basic_string() {
43        parse("foo:bar");
44    }
45
46    #[test]
47    fn parses_whitespace() {
48        let cases = [" ", "    ", "\t"];
49        for query in cases.iter() {
50            let res = parse(query);
51            assert!(
52                matches!(res, QueryNode::MatchAllDocs),
53                "Failed to parse MatchAllDocs query out of empty input"
54            );
55        }
56    }
57
58    #[test]
59    fn parses_unquoted_default_field_query() {
60        let cases = ["foo"];
61        for query in cases.iter() {
62            let res = parse(query);
63            assert!(
64                matches!(res,
65                QueryNode::AttributeTerm { ref attr, ref value }
66                if attr == DEFAULT_FIELD && value == "foo"),
67                "Unable to properly parse '{query:?}' - got {res:?}"
68            );
69        }
70    }
71
72    #[test]
73    fn parses_quoted_default_field_query() {
74        let cases = ["\"foo bar\""];
75        for query in cases.iter() {
76            let res = parse(query);
77            assert!(
78                matches!(res,
79                QueryNode::QuotedAttribute { ref attr, ref phrase }
80                if attr == DEFAULT_FIELD && phrase == "foo bar"),
81                "Unable to properly parse '{query:?}' - got {res:?}"
82            );
83        }
84    }
85
86    #[test]
87    fn parses_attribute_term_query() {
88        let cases = ["foo:bar", "foo:(bar)", "foo:b\\ar", "foo:(b\\ar)"];
89        for query in cases.iter() {
90            let res = parse(query);
91            assert!(
92                matches!(res,
93                QueryNode::AttributeTerm { ref attr, ref value }
94                if attr == "foo" && value == "bar"),
95                "Unable to properly parse '{query:?}' - got {res:?}"
96            );
97        }
98    }
99
100    #[test]
101    fn parses_numeric_attribute_term_query() {
102        let cases = ["foo:10"];
103        for query in cases.iter() {
104            let res = parse(query);
105            assert!(
106                matches!(res,
107                QueryNode::AttributeTerm { ref attr, ref value }
108                if attr == "foo" && value == "10"),
109                "Unable to properly parse '{query:?}' - got {res:?}"
110            );
111        }
112    }
113
114    #[test]
115    fn parses_attribute_term_query_with_escapes() {
116        let cases = ["foo:bar\\:baz", "fo\\o:bar\\:baz"];
117        for query in cases.iter() {
118            let res = parse(query);
119            assert!(
120                matches!(res,
121                QueryNode::AttributeTerm { ref attr, ref value }
122                if attr == "foo" && value == "bar:baz"),
123                "Unable to properly parse '{query:?}' - got {res:?}"
124            );
125        }
126    }
127
128    #[test]
129    fn parses_attribute_comparison_query_with_escapes() {
130        let cases = ["foo:<4.12345E-4", "foo:<4.12345E\\-4"];
131        for query in cases.iter() {
132            let res = parse(query);
133            assert!(
134                matches!(res,
135                QueryNode::AttributeComparison { ref attr, value: ComparisonValue::Float(ref compvalue), comparator: Comparison::Lt }
136                if attr == "foo" && (*compvalue - 4.12345E-4).abs() < 0.001),
137                "Unable to properly parse '{query:?}' - got {res:?}"
138            );
139        }
140    }
141
142    #[test]
143    fn parses_and_normalizes_multiterm_query() {
144        let cases = ["foo bar", "foo        bar"];
145        for query in cases.iter() {
146            let res = parse(query);
147            assert!(
148                matches!(res,
149                QueryNode::AttributeTerm { ref attr, ref value }
150                if attr == DEFAULT_FIELD && value == "foo bar"),
151                "Unable to properly parse '{query:?}' - got {res:?}"
152            );
153        }
154    }
155
156    #[test]
157    fn parses_multiple_multiterm_query() {
158        let cases = ["foo bar baz AND qux quux quuz"];
159        for query in cases.iter() {
160            let res = parse(query);
161            assert!(
162                matches!(res, QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
163                    matches!(nodes[0], QueryNode::AttributeTerm { ref attr, ref value } if attr == "_default_" && value == "foo bar")
164                        && matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "_default_" && value == "baz")
165                        && matches!(nodes[2], QueryNode::AttributeTerm { ref attr, ref value } if attr == "_default_" && value == "qux")
166                        && matches!(nodes[3], QueryNode::AttributeTerm { ref attr, ref value } if attr == "_default_" && value == "quux quuz")
167                ),
168                "Unable to properly parse '{query:?}' - got {res:?}"
169            );
170        }
171    }
172
173    #[test]
174    fn parses_negated_attribute_term_query() {
175        let cases = ["-foo:bar", "- foo:bar", "NOT foo:bar"];
176        for query in cases.iter() {
177            let res = parse(query);
178            if let QueryNode::NegatedNode { ref node } = res
179                && let QueryNode::AttributeTerm {
180                    ref attr,
181                    ref value,
182                } = **node
183                && attr == "foo"
184                && value == "bar"
185            {
186                continue;
187            }
188            panic!("Unable to properly parse '{query:?}' - got {res:?}")
189        }
190    }
191
192    #[test]
193    fn parses_quoted_attribute_term_query() {
194        let cases = ["foo:\"bar baz\""];
195        for query in cases.iter() {
196            let res = parse(query);
197            assert!(
198                matches!(res,
199                QueryNode::QuotedAttribute { ref attr, ref phrase }
200                if attr == "foo" && phrase == "bar baz"),
201                "Unable to properly parse '{query:?}' - got {res:?}"
202            );
203        }
204    }
205
206    #[test]
207    fn parses_attribute_prefix_query() {
208        let cases = ["foo:ba*"];
209        for query in cases.iter() {
210            let res = parse(query);
211            assert!(
212                matches!(res,
213                QueryNode::AttributePrefix { ref attr, ref prefix }
214                if attr == "foo" && prefix == "ba"), // We strip the trailing * from the prefix for escaping
215                "Unable to properly parse '{query:?}' - got {res:?}"
216            );
217        }
218    }
219
220    #[test]
221    fn parses_attribute_wildcard_query() {
222        let cases = ["foo:b*r"];
223        for query in cases.iter() {
224            let res = parse(query);
225            assert!(
226                matches!(res,
227                QueryNode::AttributeWildcard { ref attr, ref wildcard }
228                if attr == "foo" && wildcard == "b*r"),
229                "Unable to properly parse '{query:?}' - got {res:?}"
230            );
231        }
232    }
233
234    #[test]
235    fn parses_attribute_wildcard_query_with_trailing_question_mark() {
236        let cases = ["foo:ba?"];
237        for query in cases.iter() {
238            let res = parse(query);
239            assert!(
240                matches!(res,
241                QueryNode::AttributeWildcard { ref attr, ref wildcard }
242                if attr == "foo" && wildcard == "ba?"),
243                "Unable to properly parse '{query:?}' - got {res:?}"
244            );
245        }
246    }
247
248    #[test]
249    fn parses_attribute_wildcard_query_with_leading_wildcard() {
250        let cases = ["foo:*ar"];
251        for query in cases.iter() {
252            let res = parse(query);
253            assert!(
254                matches!(res,
255                QueryNode::AttributeWildcard { ref attr, ref wildcard }
256                if attr == "foo" && wildcard == "*ar"),
257                "Unable to properly parse '{query:?}' - got {res:?}"
258            );
259        }
260    }
261
262    #[test]
263    fn parses_non_numeric_attribute_comparison_query() {
264        let cases = ["foo:>=bar"];
265        for query in cases.iter() {
266            let res = parse(query);
267            assert!(
268                matches!(res,
269                QueryNode::AttributeComparison {
270                    ref attr,
271                    value: ComparisonValue::String(ref cval),
272                    comparator: Comparison::Gte
273                } if attr == "foo" && cval == "bar"),
274                "Unable to properly parse '{query:?}' - got {res:?}'"
275            );
276        }
277    }
278
279    #[test]
280    fn parses_numeric_attribute_range_query() {
281        let cases = ["foo:[10 TO 20]"];
282        for query in cases.iter() {
283            let res = parse(query);
284            assert!(
285                matches!(res,
286                QueryNode::AttributeRange {
287                    ref attr,
288                    lower: ComparisonValue::Integer(ref lstr),
289                    lower_inclusive: true,
290                    upper: ComparisonValue::Integer(ref ustr),
291                    upper_inclusive: true
292                } if attr == "foo" && *lstr == 10 && *ustr == 20),
293                "Unable to properly parse '{query:?}' - got {res:?}'"
294            );
295        }
296    }
297
298    #[test]
299    fn parses_non_numeric_attribute_range_query() {
300        let cases = ["foo:{bar TO baz}"];
301        for query in cases.iter() {
302            let res = parse(query);
303            assert!(
304                matches!(res,
305                QueryNode::AttributeRange {
306                    ref attr,
307                    lower: ComparisonValue::String(ref lstr),
308                    lower_inclusive: false,
309                    upper: ComparisonValue::String(ref ustr),
310                    upper_inclusive: false
311                } if attr == "foo" && lstr == "bar" && ustr == "baz"),
312                "Unable to properly parse '{query:?}' - got {res:?}'"
313            );
314        }
315    }
316
317    #[test]
318    fn parses_attribute_range_query_with_open_endpoints() {
319        let cases = ["foo:[* TO *]"];
320        for query in cases.iter() {
321            let res = parse(query);
322            assert!(
323                matches!(res,
324                QueryNode::AttributeRange {
325                    ref attr,
326                    lower: ComparisonValue::Unbounded,
327                    lower_inclusive: true,
328                    upper: ComparisonValue::Unbounded,
329                    upper_inclusive: true
330                } if attr == "foo"),
331                "Unable to properly parse '{query:?}' - got {res:?}'"
332            );
333        }
334    }
335
336    #[test]
337    fn parses_attribute_range_query_with_fake_wildcards() {
338        let cases = ["foo:[ba* TO b*z]"];
339        for query in cases.iter() {
340            let res = parse(query);
341            assert!(
342                matches!(res,
343                QueryNode::AttributeRange {
344                    ref attr,
345                    lower: ComparisonValue::String(ref lstr),
346                    lower_inclusive: true,
347                    upper: ComparisonValue::String(ref ustr),
348                    upper_inclusive: true
349                } if attr == "foo" && lstr == "ba*" && ustr == "b*z"),
350                "Unable to properly parse '{query:?}' - got {res:?}'"
351            );
352        }
353    }
354
355    #[test]
356    fn parses_attribute_exists_query() {
357        let cases = ["_exists_:foo", "_exists_:\"foo\""];
358        for query in cases.iter() {
359            let res = parse(query);
360            assert!(
361                matches!(res,
362                QueryNode::AttributeExists { ref attr }
363                if attr == "foo"),
364                "Unable to properly parse '{query:?}' - got {res:?}"
365            );
366        }
367    }
368
369    #[test]
370    fn parses_attribute_exists_query_with_escapes() {
371        let cases = ["_exists_:foo\\ bar"];
372        for query in cases.iter() {
373            let res = parse(query);
374            assert!(
375                matches!(res,
376                QueryNode::AttributeExists { ref attr }
377                if attr == "foo bar"),
378                "Unable to properly parse '{query:?}' - got {res:?}"
379            );
380        }
381    }
382
383    #[test]
384    fn parses_star_as_wildcard_not_exists() {
385        let cases = ["foo:*"];
386        for query in cases.iter() {
387            let res = parse(query);
388            assert!(
389                matches!(res,
390                QueryNode::AttributeWildcard { ref attr, ref wildcard }
391                if attr == "foo" && wildcard == "*"),
392                "Unable to properly parse '{query:?}' - got {res:?}"
393            );
394        }
395    }
396
397    #[test]
398    fn parses_attribute_missing_query() {
399        let cases = ["_missing_:foo", "_missing_:\"foo\""];
400        for query in cases.iter() {
401            let res = parse(query);
402            assert!(
403                matches!(res,
404                QueryNode::AttributeMissing { ref attr }
405                if attr == "foo"),
406                "Unable to properly parse '{query:?}' - got {res:?}"
407            );
408        }
409    }
410
411    #[test]
412    fn parses_attribute_missing_query_with_escapes() {
413        let cases = ["_missing_:foo\\ bar"];
414        for query in cases.iter() {
415            let res = parse(query);
416            assert!(
417                matches!(res,
418                QueryNode::AttributeMissing { ref attr }
419                if attr == "foo bar"),
420                "Unable to properly parse '{query:?}' - got {res:?}"
421            );
422        }
423    }
424
425    #[test]
426    fn parses_match_all_docs_query() {
427        let cases = ["*:*", "*", "_default_:*", "foo:(*:*)"];
428        for query in cases.iter() {
429            let res = parse(query);
430            assert!(
431                matches!(res, QueryNode::MatchAllDocs),
432                "Failed to parse '{query:?}' as MatchAllDocs, got {res:?}"
433            );
434        }
435    }
436
437    #[test]
438    fn parses_all_as_wildcard() {
439        let cases = ["_all:*"];
440        for query in cases.iter() {
441            let res = parse(query);
442            assert!(
443                matches!(res,
444                QueryNode::AttributeWildcard { ref attr, ref wildcard }
445                if attr == "_all" && wildcard == "*"),
446                "Unable to properly parse '{query:?}' - got {res:?}"
447            );
448        }
449    }
450
451    #[test]
452    fn parses_match_no_docs_query() {
453        let cases = [
454            "NOT *:*",
455            "NOT *",
456            "NOT _default_:*",
457            "NOT foo:(*:*)",
458            "foo:(NOT *:*)",
459        ];
460        for query in cases.iter() {
461            let res = parse(query);
462            assert!(
463                matches!(res, QueryNode::MatchNoDocs),
464                "Failed to parse '{query:?}' as MatchNoDocs, got {res:?}"
465            );
466        }
467    }
468
469    #[test]
470    fn parses_boolean_nodes_with_implicit_operators() {
471        let cases = ["foo:bar baz:qux quux:quuz"];
472        for query in cases.iter() {
473            let res = parse(query);
474            assert!(
475                matches!(res, QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
476                    matches!(nodes[0], QueryNode::AttributeTerm { ref attr, ref value } if attr == "foo" && value == "bar")
477                        && matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "baz" && value == "qux")
478                        && matches!(nodes[2], QueryNode::AttributeTerm { ref attr, ref value } if attr == "quux" && value == "quuz")
479                ),
480                "Unable to properly parse '{query:?}' - got {res:?}"
481            );
482        }
483    }
484
485    #[test]
486    fn parses_boolean_nodes_with_implicit_operators_and_negated_clauses() {
487        let cases = [
488            "NOT foo:bar baz:qux NOT quux:quuz",
489            "NOT foo:bar baz:qux -quux:quuz",
490            "-foo:bar baz:qux NOT quux:quuz",
491            "-foo:bar baz:qux -quux:quuz",
492        ];
493        for query in cases.iter() {
494            let res = parse(query);
495            assert!(
496                matches!(res, QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
497                    matches!(nodes[0], QueryNode::NegatedNode { ref node } if matches!(**node, QueryNode::AttributeTerm {ref attr, ref value } if attr == "foo" && value == "bar"))
498                        && matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "baz" && value == "qux")
499                        && matches!(nodes[2], QueryNode::NegatedNode { ref node } if matches!(**node, QueryNode::AttributeTerm {ref attr, ref value } if attr == "quux" && value == "quuz"))
500                ),
501                "Unable to properly parse '{query:?}' - got {res:?}"
502            );
503        }
504    }
505
506    #[test]
507    fn parses_boolean_nodes_with_or_not() {
508        let cases = [
509            "foo:bar OR -baz:qux quux:quuz",
510            "foo:bar OR NOT baz:qux quux:quuz",
511            "foo:bar OR -baz:qux AND quux:quuz",
512            "foo:bar OR NOT baz:qux AND quux:quuz",
513        ];
514        for query in cases.iter() {
515            let res = parse(query);
516            assert!(
517                matches!(res, QueryNode::Boolean { oper: BooleanType::Or, ref nodes } if
518                    matches!(nodes[0], QueryNode::AttributeTerm {ref attr, ref value } if attr == "foo" && value == "bar")
519                        && matches!(nodes[1], QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
520                            matches!(nodes[0], QueryNode::NegatedNode { ref node } if matches!(**node, QueryNode::AttributeTerm {ref attr, ref value } if attr == "baz" && value == "qux") &&
521                            matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "quux" && value == "quuz"))
522                        )
523                ),
524                "Unable to properly parse '{query:?}' - got {res:?}"
525            );
526        }
527    }
528
529    #[test]
530    fn parses_boolean_nodes_with_implicit_or_explicit_operators() {
531        let cases = [
532            "foo:bar OR baz:qux quux:quuz",
533            "foo:bar || baz:qux quux:quuz",
534            "foo:bar OR baz:qux AND quux:quuz",
535            "foo:bar || baz:qux && quux:quuz",
536        ];
537        for query in cases.iter() {
538            let res = parse(query);
539            assert!(
540                matches!(res, QueryNode::Boolean { oper: BooleanType::Or, ref nodes } if
541                    matches!(nodes[0], QueryNode::AttributeTerm {ref attr, ref value } if attr == "foo" && value == "bar")
542                        && matches!(nodes[1], QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
543                            matches!(nodes[0], QueryNode::AttributeTerm { ref attr, ref value } if attr == "baz" && value == "qux") &&
544                            matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "quux" && value == "quuz")
545                        )
546                ),
547                "Unable to properly parse '{query:?}' - got {res:?}"
548            );
549        }
550    }
551
552    #[test]
553    fn parses_nested_boolean_query_node() {
554        let cases = ["foo:bar (baz:qux quux:quuz)"];
555        for query in cases.iter() {
556            let res = parse(query);
557            assert!(
558                matches!(res, QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
559                    matches!(nodes[0], QueryNode::AttributeTerm {ref attr, ref value } if attr == "foo" && value == "bar")
560                        && matches!(nodes[1], QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
561                            matches!(nodes[0], QueryNode::AttributeTerm { ref attr, ref value } if attr == "baz" && value == "qux") &&
562                            matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "quux" && value == "quuz")
563                        )
564                ),
565                "Unable to properly parse '{query:?}' - got {res:?}"
566            );
567        }
568    }
569
570    #[test]
571    fn parses_nested_boolean_query_node_with_or() {
572        let cases = ["(foo:bar OR baz:qux) quux:quuz"];
573        for query in cases.iter() {
574            let res = parse(query);
575
576            assert!(
577                matches!(res, QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
578                    matches!(nodes[0], QueryNode::Boolean { oper: BooleanType::Or, ref nodes } if
579                        matches!(nodes[0], QueryNode::AttributeTerm { ref attr, ref value } if attr == "foo" && value == "bar") &&
580                        matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == "baz" && value == "qux")
581                    ) && matches!(nodes[1], QueryNode::AttributeTerm {ref attr, ref value } if attr == "quux" && value == "quuz")),
582                "Unable to properly parse '{query:?}' - got {res:?}"
583            );
584        }
585    }
586
587    #[test]
588    fn parses_negated_parenthesized_default_multiterm_query() {
589        let cases = ["NOT (foo bar)", "-(foo bar)"];
590        for query in cases.iter() {
591            let res = parse(query);
592            if let QueryNode::NegatedNode { ref node } = res
593                && let QueryNode::AttributeTerm {
594                    ref attr,
595                    ref value,
596                } = **node
597                && attr == DEFAULT_FIELD
598                && value == "foo bar"
599            {
600                continue;
601            }
602            panic!("Unable to properly parse '{query:?}' - got {res:?}")
603        }
604    }
605
606    #[test]
607    fn parses_multiterm_with_leading_not_without_parens() {
608        let cases = ["NOT foo bar", "- foo bar"]; // NOT only applies to the first term
609        for query in cases.iter() {
610            let res = parse(query);
611
612            assert!(
613                matches!(res, QueryNode::Boolean { oper: BooleanType::And, ref nodes } if
614                    matches!(nodes[0], QueryNode::NegatedNode { ref node } if matches!(**node, QueryNode::AttributeTerm { ref attr, ref value } if attr == DEFAULT_FIELD && value == "foo"))
615                        && matches!(nodes[1], QueryNode::AttributeTerm { ref attr, ref value } if attr == DEFAULT_FIELD && value == "bar")),
616                "Unable to properly parse '{query:?}' - got {res:?}"
617            );
618        }
619    }
620
621    #[test]
622    fn parses_negated_parenthesized_fielded_multiterm_query() {
623        let cases = ["NOT foo:(bar baz)", "-foo:(bar baz)"];
624        for query in cases.iter() {
625            let res = parse(query);
626            if let QueryNode::NegatedNode { ref node } = res
627                && let QueryNode::AttributeTerm {
628                    ref attr,
629                    ref value,
630                } = **node
631                && attr == "foo"
632                && value == "bar baz"
633            {
634                continue;
635            }
636            panic!("Unable to properly parse '{query:?}' - got {res:?}")
637        }
638    }
639}
vrl/datadog/search/parser.rs

vrl/datadog/search/
parser.rs
