vector_core/schema/
definition.rs

1use std::collections::{BTreeMap, BTreeSet};
2
3use lookup::{
4    OwnedTargetPath, OwnedValuePath, PathPrefix, lookup_v2::TargetPath, owned_value_path,
5};
6use vrl::value::{Kind, kind::Collection};
7
8use crate::config::{LegacyKey, LogNamespace, log_schema};
9
10/// The definition of a schema.
11///
12/// This struct contains all the information needed to inspect the schema of an event emitted by
13/// a source/transform.
14#[derive(Clone, Debug, PartialEq, PartialOrd)]
15pub struct Definition {
16    /// The type of the event
17    event_kind: Kind,
18
19    /// The type of the metadata.
20    metadata_kind: Kind,
21
22    /// Semantic meaning assigned to fields within the collection.
23    ///
24    /// The value within this map points to a path inside the `event_kind`.
25    /// Meanings currently can't point to metadata.
26    meaning: BTreeMap<String, MeaningPointer>,
27
28    /// Type definitions of components can change depending on the log namespace chosen.
29    /// This records which ones are possible.
30    /// An empty set means the definition can't be for a log
31    log_namespaces: BTreeSet<LogNamespace>,
32}
33
34/// In regular use, a semantic meaning points to exactly _one_ location in the collection. However,
35/// when merging two [`Definition`]s, we need to be able to allow for two definitions with the same
36/// semantic meaning identifier to be merged together.
37///
38/// We cannot error when this happens, because a follow-up component (such as the `remap`
39/// transform) might rectify the issue of having a semantic meaning with multiple pointers.
40///
41/// Because of this, we encapsulate this state in an enum. The schema validation step done by the
42/// sink builder, will return an error if the definition stores an "invalid" meaning pointer.
43#[derive(Clone, Debug, PartialEq, PartialOrd)]
44enum MeaningPointer {
45    Valid(OwnedTargetPath),
46    Invalid(BTreeSet<OwnedTargetPath>),
47}
48
49impl MeaningPointer {
50    fn merge(self, other: Self) -> Self {
51        let set = match (self, other) {
52            (Self::Valid(lhs), Self::Valid(rhs)) if lhs == rhs => return Self::Valid(lhs),
53            (Self::Valid(lhs), Self::Valid(rhs)) => BTreeSet::from([lhs, rhs]),
54            (Self::Valid(lhs), Self::Invalid(mut rhs)) => {
55                rhs.insert(lhs);
56                rhs
57            }
58            (Self::Invalid(mut lhs), Self::Valid(rhs)) => {
59                lhs.insert(rhs);
60                lhs
61            }
62            (Self::Invalid(mut lhs), Self::Invalid(rhs)) => {
63                lhs.extend(rhs);
64                lhs
65            }
66        };
67
68        Self::Invalid(set)
69    }
70}
71
72impl Definition {
73    /// The most general possible definition. The `Kind` is `any`, and all `log_namespaces` are enabled.
74    pub fn any() -> Self {
75        Self {
76            event_kind: Kind::any(),
77            metadata_kind: Kind::any(),
78            meaning: BTreeMap::default(),
79            log_namespaces: [LogNamespace::Legacy, LogNamespace::Vector].into(),
80        }
81    }
82
83    /// Creates a new definition that is of the event kind specified, and an empty object for metadata.
84    /// There are no meanings.
85    /// The `log_namespaces` are used to list the possible namespaces the schema is for.
86    pub fn new_with_default_metadata(
87        event_kind: Kind,
88        log_namespaces: impl Into<BTreeSet<LogNamespace>>,
89    ) -> Self {
90        Self {
91            event_kind,
92            metadata_kind: Kind::object(Collection::any()),
93            meaning: BTreeMap::default(),
94            log_namespaces: log_namespaces.into(),
95        }
96    }
97
98    /// Creates a new definition, specifying both the event and metadata kind.
99    /// There are no meanings.
100    /// The `log_namespaces` are used to list the possible namespaces the schema is for.
101    pub fn new(
102        event_kind: Kind,
103        metadata_kind: Kind,
104        log_namespaces: impl Into<BTreeSet<LogNamespace>>,
105    ) -> Self {
106        Self {
107            event_kind,
108            metadata_kind,
109            meaning: BTreeMap::default(),
110            log_namespaces: log_namespaces.into(),
111        }
112    }
113
114    /// An object with any fields, and the `Legacy` namespace.
115    /// This is the default schema for a source that does not explicitly provide one yet.
116    pub fn default_legacy_namespace() -> Self {
117        Self::new_with_default_metadata(Kind::any_object(), [LogNamespace::Legacy])
118    }
119
120    /// An object with no fields, and the `Legacy` namespace.
121    /// This is what most sources use for the legacy namespace.
122    pub fn empty_legacy_namespace() -> Self {
123        Self::new_with_default_metadata(Kind::object(Collection::empty()), [LogNamespace::Legacy])
124    }
125
126    /// Returns the source schema for a source that produce the listed log namespaces,
127    /// but an explicit schema was not provided.
128    pub fn default_for_namespace(log_namespaces: &BTreeSet<LogNamespace>) -> Self {
129        let is_legacy = log_namespaces.contains(&LogNamespace::Legacy);
130        let is_vector = log_namespaces.contains(&LogNamespace::Vector);
131        match (is_legacy, is_vector) {
132            (false, false) => Self::new_with_default_metadata(Kind::any(), []),
133            (true, false) => Self::default_legacy_namespace(),
134            (false, true) => Self::new_with_default_metadata(Kind::any(), [LogNamespace::Vector]),
135            (true, true) => Self::any(),
136        }
137    }
138
139    /// The set of possible log namespaces that events can use. When merged, this is the union of all inputs.
140    pub fn log_namespaces(&self) -> &BTreeSet<LogNamespace> {
141        &self.log_namespaces
142    }
143
144    /// Adds the `source_type` and `ingest_timestamp` metadata fields, which are added to every Vector source.
145    /// This function should be called in the same order as the values are actually inserted into the event.
146    #[must_use]
147    pub fn with_standard_vector_source_metadata(self) -> Self {
148        self.with_vector_metadata(
149            log_schema().source_type_key(),
150            &owned_value_path!("source_type"),
151            Kind::bytes(),
152            None,
153        )
154        .with_vector_metadata(
155            log_schema().timestamp_key(),
156            &owned_value_path!("ingest_timestamp"),
157            Kind::timestamp(),
158            None,
159        )
160    }
161
162    /// This should be used wherever `LogNamespace::insert_source_metadata` is used to insert metadata.
163    /// This automatically detects which log namespaces are used, and also automatically
164    /// determines if there are possible conflicts from existing field names (usually from the selected decoder).
165    /// This function should be called in the same order as the values are actually inserted into the event.
166    #[must_use]
167    pub fn with_source_metadata(
168        self,
169        source_name: &str,
170        legacy_path: Option<LegacyKey<OwnedValuePath>>,
171        vector_path: &OwnedValuePath,
172        kind: Kind,
173        meaning: Option<&str>,
174    ) -> Self {
175        self.with_namespaced_metadata(source_name, legacy_path, vector_path, kind, meaning)
176    }
177
178    /// This should be used wherever `LogNamespace::insert_vector_metadata` is used to insert metadata.
179    /// This automatically detects which log namespaces are used, and also automatically
180    /// determines if there are possible conflicts from existing field names (usually from the selected decoder).
181    /// This function should be called in the same order as the values are actually inserted into the event.
182    #[must_use]
183    pub fn with_vector_metadata(
184        self,
185        legacy_path: Option<&OwnedValuePath>,
186        vector_path: &OwnedValuePath,
187        kind: Kind,
188        meaning: Option<&str>,
189    ) -> Self {
190        self.with_namespaced_metadata(
191            "vector",
192            legacy_path.cloned().map(LegacyKey::InsertIfEmpty),
193            vector_path,
194            kind,
195            meaning,
196        )
197    }
198
199    /// This generalizes the `LogNamespace::insert_*` methods for type definitions.
200    /// This assumes the legacy key is either guaranteed to not collide or is inserted with `try_insert`.
201    fn with_namespaced_metadata(
202        self,
203        prefix: &str,
204        legacy_path: Option<LegacyKey<OwnedValuePath>>,
205        vector_path: &OwnedValuePath,
206        kind: Kind,
207        meaning: Option<&str>,
208    ) -> Self {
209        let legacy_definition = legacy_path.and_then(|legacy_path| {
210            if self.log_namespaces.contains(&LogNamespace::Legacy) {
211                match legacy_path {
212                    LegacyKey::InsertIfEmpty(legacy_path) => Some(self.clone().try_with_field(
213                        &legacy_path,
214                        kind.clone(),
215                        meaning,
216                    )),
217                    LegacyKey::Overwrite(legacy_path) => Some(self.clone().with_event_field(
218                        &legacy_path,
219                        kind.clone(),
220                        meaning,
221                    )),
222                }
223            } else {
224                None
225            }
226        });
227
228        let vector_definition = if self.log_namespaces.contains(&LogNamespace::Vector) {
229            Some(self.clone().with_metadata_field(
230                &vector_path.with_field_prefix(prefix),
231                kind,
232                meaning,
233            ))
234        } else {
235            None
236        };
237
238        match (legacy_definition, vector_definition) {
239            (Some(a), Some(b)) => a.merge(b),
240            (Some(x), _) | (_, Some(x)) => x,
241            (None, None) => self,
242        }
243    }
244
245    /// Add type information for an event or metadata field.
246    /// A non-root required field means the root type must be an object, so the type will be automatically
247    /// restricted to an object.
248    ///
249    /// # Panics
250    /// - If the path is not root, and the definition does not allow the type to be an object.
251    #[must_use]
252    pub fn with_field(
253        self,
254        target_path: &OwnedTargetPath,
255        kind: Kind,
256        meaning: Option<&str>,
257    ) -> Self {
258        match target_path.prefix {
259            PathPrefix::Event => self.with_event_field(&target_path.path, kind, meaning),
260            PathPrefix::Metadata => self.with_metadata_field(&target_path.path, kind, meaning),
261        }
262    }
263
264    /// Add type information for an event field.
265    /// A non-root required field means the root type must be an object, so the type will be automatically
266    /// restricted to an object.
267    ///
268    /// # Panics
269    /// - If the path is not root, and the definition does not allow the type to be an object.
270    /// - Provided path has one or more coalesced segments (e.g. `.(foo | bar)`).
271    #[must_use]
272    pub fn with_event_field(
273        mut self,
274        path: &OwnedValuePath,
275        kind: Kind,
276        meaning: Option<&str>,
277    ) -> Self {
278        if !path.is_root() {
279            assert!(
280                self.event_kind.as_object().is_some(),
281                "Setting a field on a value that cannot be an object"
282            );
283        }
284
285        self.event_kind.set_at_path(path, kind);
286
287        if let Some(meaning) = meaning {
288            self.meaning.insert(
289                meaning.to_owned(),
290                MeaningPointer::Valid(OwnedTargetPath::event(path.clone())),
291            );
292        }
293
294        self
295    }
296
297    /// Add type information for an event field.
298    /// This inserts type information similar to `LogEvent::try_insert`.
299    #[must_use]
300    pub fn try_with_field(
301        mut self,
302        path: &OwnedValuePath,
303        kind: Kind,
304        meaning: Option<&str>,
305    ) -> Self {
306        let existing_type = self.event_kind.at_path(path);
307
308        if existing_type.is_undefined() {
309            // Guaranteed to never be set, so the insertion will always succeed.
310            self.with_event_field(path, kind, meaning)
311        } else if !existing_type.contains_undefined() {
312            // Guaranteed to always be set (or is never), so the insertion will always fail.
313            self
314        } else {
315            // Not sure if the insertion will be successful. The type definition should contain both
316            // possibilities. The meaning is not set, since it can't be relied on.
317
318            let success_definition = self.clone().with_event_field(path, kind, None);
319            // If the existing type contains `undefined`, the new type will always be used, so remove it.
320            self.event_kind
321                .set_at_path(path, existing_type.without_undefined());
322            self.merge(success_definition)
323        }
324    }
325
326    /// Add type information for an event field.
327    /// A non-root required field means the root type must be an object, so the type will be automatically
328    /// restricted to an object.
329    ///
330    /// # Panics
331    /// - If the path is not root, and the definition does not allow the type to be an object
332    /// - Provided path has one or more coalesced segments (e.g. `.(foo | bar)`).
333    #[must_use]
334    pub fn with_metadata_field(
335        mut self,
336        path: &OwnedValuePath,
337        kind: Kind,
338        meaning: Option<&str>,
339    ) -> Self {
340        if !path.is_root() {
341            assert!(
342                self.metadata_kind.as_object().is_some(),
343                "Setting a field on a value that cannot be an object"
344            );
345        }
346
347        self.metadata_kind.set_at_path(path, kind);
348
349        if let Some(meaning) = meaning {
350            self.meaning.insert(
351                meaning.to_owned(),
352                MeaningPointer::Valid(OwnedTargetPath::metadata(path.clone())),
353            );
354        }
355
356        self
357    }
358
359    /// Add type information for an optional event field.
360    ///
361    /// # Panics
362    ///
363    /// See `Definition::require_field`.
364    #[must_use]
365    pub fn optional_field(self, path: &OwnedValuePath, kind: Kind, meaning: Option<&str>) -> Self {
366        self.with_event_field(path, kind.or_undefined(), meaning)
367    }
368
369    /// Register a semantic meaning for the definition.
370    ///
371    /// # Panics
372    ///
373    /// This method panics if the provided path points to an unknown location in the collection.
374    #[must_use]
375    pub fn with_meaning(mut self, target_path: OwnedTargetPath, meaning: &str) -> Self {
376        self.add_meaning(target_path, meaning);
377        self
378    }
379
380    /// Adds the meaning pointing to the given path to our list of meanings.
381    ///
382    /// # Panics
383    ///
384    /// This method panics if the provided path points to an unknown location in the collection.
385    pub fn add_meaning(&mut self, target_path: OwnedTargetPath, meaning: &str) {
386        self.try_with_meaning(target_path, meaning)
387            .unwrap_or_else(|err| panic!("{}", err));
388    }
389
390    /// Register a semantic meaning for the definition.
391    ///
392    /// # Errors
393    ///
394    /// Returns an error if the provided path points to an unknown location in the collection.
395    pub fn try_with_meaning(
396        &mut self,
397        target_path: OwnedTargetPath,
398        meaning: &str,
399    ) -> Result<(), &'static str> {
400        match target_path.prefix {
401            PathPrefix::Event
402                if !self
403                    .event_kind
404                    .at_path(&target_path.path)
405                    .contains_any_defined() =>
406            {
407                Err("meaning must point to a valid path")
408            }
409
410            PathPrefix::Metadata
411                if !self
412                    .metadata_kind
413                    .at_path(&target_path.path)
414                    .contains_any_defined() =>
415            {
416                Err("meaning must point to a valid path")
417            }
418
419            _ => {
420                self.meaning
421                    .insert(meaning.to_owned(), MeaningPointer::Valid(target_path));
422                Ok(())
423            }
424        }
425    }
426
427    /// Set the kind for all unknown fields.
428    #[must_use]
429    pub fn unknown_fields(mut self, unknown: impl Into<Kind>) -> Self {
430        let unknown = unknown.into();
431        if let Some(object) = self.event_kind.as_object_mut() {
432            object.set_unknown(unknown.clone());
433        }
434        if let Some(array) = self.event_kind.as_array_mut() {
435            array.set_unknown(unknown);
436        }
437        self
438    }
439
440    /// Merge `other` definition into `self`.
441    ///
442    /// This just takes the union of both definitions.
443    #[must_use]
444    pub fn merge(mut self, mut other: Self) -> Self {
445        for (other_id, other_meaning) in other.meaning {
446            let meaning = match self.meaning.remove(&other_id) {
447                Some(this_meaning) => this_meaning.merge(other_meaning),
448                None => other_meaning,
449            };
450
451            self.meaning.insert(other_id, meaning);
452        }
453
454        self.event_kind = self.event_kind.union(other.event_kind);
455        self.metadata_kind = self.metadata_kind.union(other.metadata_kind);
456        self.log_namespaces.append(&mut other.log_namespaces);
457        self
458    }
459
460    /// If the schema definition depends on the `LogNamespace`, this combines the individual
461    /// definitions for each `LogNamespace`.
462    pub fn combine_log_namespaces(
463        log_namespaces: &BTreeSet<LogNamespace>,
464        legacy: Self,
465        vector: Self,
466    ) -> Self {
467        let mut combined =
468            Definition::new_with_default_metadata(Kind::never(), log_namespaces.clone());
469
470        if log_namespaces.contains(&LogNamespace::Legacy) {
471            combined = combined.merge(legacy);
472        }
473        if log_namespaces.contains(&LogNamespace::Vector) {
474            combined = combined.merge(vector);
475        }
476        combined
477    }
478
479    /// Returns an `OwnedTargetPath` into an event, based on the provided `meaning`, if the meaning exists.
480    pub fn meaning_path(&self, meaning: &str) -> Option<&OwnedTargetPath> {
481        match self.meaning.get(meaning) {
482            Some(MeaningPointer::Valid(path)) => Some(path),
483            None | Some(MeaningPointer::Invalid(_)) => None,
484        }
485    }
486
487    pub fn invalid_meaning(&self, meaning: &str) -> Option<&BTreeSet<OwnedTargetPath>> {
488        match &self.meaning.get(meaning) {
489            Some(MeaningPointer::Invalid(paths)) => Some(paths),
490            None | Some(MeaningPointer::Valid(_)) => None,
491        }
492    }
493
494    pub fn meanings(&self) -> impl Iterator<Item = (&String, &OwnedTargetPath)> {
495        self.meaning
496            .iter()
497            .filter_map(|(id, pointer)| match pointer {
498                MeaningPointer::Valid(path) => Some((id, path)),
499                MeaningPointer::Invalid(_) => None,
500            })
501    }
502
503    /// Adds the meanings provided by an iterator over the given meanings.
504    ///
505    /// # Panics
506    ///
507    /// This method panics if the provided path from any of the incoming meanings point to
508    /// an unknown location in the collection.
509    pub fn add_meanings<'a>(
510        &'a mut self,
511        meanings: impl Iterator<Item = (&'a String, &'a OwnedTargetPath)>,
512    ) {
513        for (meaning, path) in meanings {
514            self.add_meaning(path.clone(), meaning);
515        }
516    }
517
518    pub fn event_kind(&self) -> &Kind {
519        &self.event_kind
520    }
521
522    pub fn event_kind_mut(&mut self) -> &mut Kind {
523        &mut self.event_kind
524    }
525
526    pub fn metadata_kind(&self) -> &Kind {
527        &self.metadata_kind
528    }
529
530    #[allow(clippy::needless_pass_by_value)]
531    pub fn kind_at<'a>(&self, target_path: impl TargetPath<'a>) -> Kind {
532        match target_path.prefix() {
533            PathPrefix::Event => self.event_kind.at_path(target_path.value_path()),
534            PathPrefix::Metadata => self.metadata_kind.at_path(target_path.value_path()),
535        }
536    }
537}
538
539#[cfg(any(test, feature = "test"))]
540mod test_utils {
541    use super::{Definition, Kind};
542    use crate::event::{Event, LogEvent};
543
544    impl Definition {
545        /// Checks that the schema definition is _valid_ for the given event.
546        ///
547        /// # Errors
548        ///
549        /// If the definition is not valid, debug info will be returned.
550        pub fn is_valid_for_event(&self, event: &Event) -> Result<(), String> {
551            if let Some(log) = event.maybe_as_log() {
552                let log: &LogEvent = log;
553
554                let actual_kind = Kind::from(log.value());
555                if let Err(path) = self.event_kind.is_superset(&actual_kind) {
556                    return Result::Err(format!(
557                        "Event value doesn't match at path: {}\n\nEvent type at path = {:?}\n\nDefinition at path = {:?}",
558                        path,
559                        actual_kind.at_path(&path).debug_info(),
560                        self.event_kind.at_path(&path).debug_info()
561                    ));
562                }
563
564                let actual_metadata_kind = Kind::from(log.metadata().value());
565                if let Err(path) = self.metadata_kind.is_superset(&actual_metadata_kind) {
566                    // return Result::Err(format!("Event metadata doesn't match definition.\n\nDefinition type=\n{:?}\n\nActual event metadata type=\n{:?}\n",
567                    //                            self.metadata_kind.debug_info(), actual_metadata_kind.debug_info()));
568                    return Result::Err(format!(
569                        "Event METADATA value doesn't match at path: {}\n\nMetadata type at path = {:?}\n\nDefinition at path = {:?}",
570                        path,
571                        actual_metadata_kind.at_path(&path).debug_info(),
572                        self.metadata_kind.at_path(&path).debug_info()
573                    ));
574                }
575                if !self.log_namespaces.contains(&log.namespace()) {
576                    return Result::Err(format!(
577                        "Event uses the {:?} LogNamespace, but the definition only contains: {:?}",
578                        log.namespace(),
579                        self.log_namespaces
580                    ));
581                }
582
583                Ok(())
584            } else {
585                // schema definitions currently only apply to logs
586                Ok(())
587            }
588        }
589
590        /// Asserts that the schema definition is _valid_ for the given event.
591        ///
592        /// # Panics
593        ///
594        /// If the definition is not valid for the event.
595        pub fn assert_valid_for_event(&self, event: &Event) {
596            if let Err(err) = self.is_valid_for_event(event) {
597                panic!("Schema definition assertion failed: {err}");
598            }
599        }
600
601        /// Asserts that the schema definition is _invalid_ for the given event.
602        ///
603        /// # Panics
604        ///
605        /// If the definition is valid for the event.
606        pub fn assert_invalid_for_event(&self, event: &Event) {
607            assert!(
608                self.is_valid_for_event(event).is_err(),
609                "Schema definition assertion should not be valid"
610            );
611        }
612    }
613}
614
615#[cfg(test)]
616mod tests {
617    use std::collections::{BTreeMap, HashMap};
618
619    use lookup::{lookup_v2::parse_target_path, owned_value_path};
620    use vrl::value::Value;
621
622    use super::*;
623    use crate::event::{Event, EventMetadata, LogEvent};
624
625    #[test]
626    fn test_definition_validity() {
627        struct TestCase {
628            title: &'static str,
629            definition: Definition,
630            event: Event,
631            valid: bool,
632        }
633
634        for TestCase {
635            title,
636            definition,
637            event,
638            valid,
639        } in [
640            TestCase {
641                title: "match",
642                definition: Definition::new(Kind::any(), Kind::any(), [LogNamespace::Legacy]),
643                event: Event::Log(LogEvent::from(BTreeMap::new())),
644                valid: true,
645            },
646            TestCase {
647                title: "event mismatch",
648                definition: Definition::new(
649                    Kind::object(Collection::empty()),
650                    Kind::any(),
651                    [LogNamespace::Legacy],
652                ),
653                event: Event::Log(LogEvent::from(BTreeMap::from([("foo".into(), 4.into())]))),
654                valid: false,
655            },
656            TestCase {
657                title: "metadata mismatch",
658                definition: Definition::new(
659                    Kind::any(),
660                    Kind::object(Collection::empty()),
661                    [LogNamespace::Legacy],
662                ),
663                event: Event::Log(LogEvent::from_parts(
664                    Value::Object(BTreeMap::new()),
665                    EventMetadata::default_with_value(
666                        BTreeMap::from([("foo".into(), 4.into())]).into(),
667                    ),
668                )),
669                valid: false,
670            },
671            TestCase {
672                title: "wrong log namespace",
673                definition: Definition::new(Kind::any(), Kind::any(), []),
674                event: Event::Log(LogEvent::from(BTreeMap::new())),
675                valid: false,
676            },
677            TestCase {
678                title: "event mismatch - null vs undefined",
679                definition: Definition::new(
680                    Kind::object(Collection::empty()),
681                    Kind::any(),
682                    [LogNamespace::Legacy],
683                ),
684                event: Event::Log(LogEvent::from(BTreeMap::from([(
685                    "foo".into(),
686                    Value::Null,
687                )]))),
688                valid: false,
689            },
690        ] {
691            let result = definition.is_valid_for_event(&event);
692            assert_eq!(result.is_ok(), valid, "{title}");
693        }
694    }
695
696    #[test]
697    fn test_empty_legacy_field() {
698        let definition = Definition::default_legacy_namespace().with_vector_metadata(
699            Some(&owned_value_path!()),
700            &owned_value_path!(),
701            Kind::integer(),
702            None,
703        );
704
705        // adding empty string legacy key doesn't change the definition (insertion will never succeed)
706        assert_eq!(definition, Definition::default_legacy_namespace());
707    }
708
709    #[test]
710    fn test_required_field() {
711        struct TestCase {
712            path: OwnedValuePath,
713            kind: Kind,
714            meaning: Option<&'static str>,
715            want: Definition,
716        }
717
718        for (
719            title,
720            TestCase {
721                path,
722                kind,
723                meaning,
724                want,
725            },
726        ) in HashMap::from([
727            (
728                "simple",
729                TestCase {
730                    path: owned_value_path!("foo"),
731                    kind: Kind::boolean(),
732                    meaning: Some("foo_meaning"),
733                    want: Definition {
734                        event_kind: Kind::object(BTreeMap::from([("foo".into(), Kind::boolean())])),
735                        metadata_kind: Kind::object(Collection::empty()),
736                        meaning: [(
737                            "foo_meaning".to_owned(),
738                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
739                        )]
740                        .into(),
741                        log_namespaces: BTreeSet::new(),
742                    },
743                },
744            ),
745            (
746                "nested fields",
747                TestCase {
748                    path: owned_value_path!("foo", "bar"),
749                    kind: Kind::regex().or_null(),
750                    meaning: Some("foobar"),
751                    want: Definition {
752                        event_kind: Kind::object(BTreeMap::from([(
753                            "foo".into(),
754                            Kind::object(BTreeMap::from([("bar".into(), Kind::regex().or_null())])),
755                        )])),
756                        metadata_kind: Kind::object(Collection::empty()),
757                        meaning: [(
758                            "foobar".to_owned(),
759                            MeaningPointer::Valid(parse_target_path(".foo.bar").unwrap()),
760                        )]
761                        .into(),
762                        log_namespaces: BTreeSet::new(),
763                    },
764                },
765            ),
766            (
767                "no meaning",
768                TestCase {
769                    path: owned_value_path!("foo"),
770                    kind: Kind::boolean(),
771                    meaning: None,
772                    want: Definition {
773                        event_kind: Kind::object(BTreeMap::from([("foo".into(), Kind::boolean())])),
774                        metadata_kind: Kind::object(Collection::empty()),
775                        meaning: BTreeMap::default(),
776                        log_namespaces: BTreeSet::new(),
777                    },
778                },
779            ),
780        ]) {
781            let got = Definition::empty_legacy_namespace().with_event_field(&path, kind, meaning);
782            assert_eq!(got.event_kind(), want.event_kind(), "{title}");
783        }
784    }
785
786    #[test]
787    fn test_optional_field() {
788        struct TestCase {
789            path: OwnedValuePath,
790            kind: Kind,
791            meaning: Option<&'static str>,
792            want: Definition,
793        }
794
795        for (
796            title,
797            TestCase {
798                path,
799                kind,
800                meaning,
801                want,
802            },
803        ) in [
804            (
805                "simple",
806                TestCase {
807                    path: owned_value_path!("foo"),
808                    kind: Kind::boolean(),
809                    meaning: Some("foo_meaning"),
810                    want: Definition {
811                        event_kind: Kind::object(BTreeMap::from([(
812                            "foo".into(),
813                            Kind::boolean().or_undefined(),
814                        )])),
815                        metadata_kind: Kind::object(Collection::any()),
816                        meaning: [(
817                            "foo_meaning".to_owned(),
818                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
819                        )]
820                        .into(),
821                        log_namespaces: BTreeSet::new(),
822                    },
823                },
824            ),
825            (
826                "nested fields",
827                TestCase {
828                    path: owned_value_path!("foo", "bar"),
829                    kind: Kind::regex().or_null(),
830                    meaning: Some("foobar"),
831                    want: Definition {
832                        event_kind: Kind::object(BTreeMap::from([(
833                            "foo".into(),
834                            Kind::object(BTreeMap::from([(
835                                "bar".into(),
836                                Kind::regex().or_null().or_undefined(),
837                            )])),
838                        )])),
839                        metadata_kind: Kind::object(Collection::any()),
840                        meaning: [(
841                            "foobar".to_owned(),
842                            MeaningPointer::Valid(parse_target_path(".foo.bar").unwrap()),
843                        )]
844                        .into(),
845                        log_namespaces: BTreeSet::new(),
846                    },
847                },
848            ),
849            (
850                "no meaning",
851                TestCase {
852                    path: owned_value_path!("foo"),
853                    kind: Kind::boolean(),
854                    meaning: None,
855                    want: Definition {
856                        event_kind: Kind::object(BTreeMap::from([(
857                            "foo".into(),
858                            Kind::boolean().or_undefined(),
859                        )])),
860                        metadata_kind: Kind::object(Collection::any()),
861                        meaning: BTreeMap::default(),
862                        log_namespaces: BTreeSet::new(),
863                    },
864                },
865            ),
866        ] {
867            let mut got = Definition::new_with_default_metadata(Kind::object(BTreeMap::new()), []);
868            got = got.optional_field(&path, kind, meaning);
869
870            assert_eq!(got, want, "{title}");
871        }
872    }
873
874    #[test]
875    fn test_unknown_fields() {
876        let want = Definition {
877            event_kind: Kind::object(Collection::from_unknown(Kind::bytes().or_integer())),
878            metadata_kind: Kind::object(Collection::any()),
879            meaning: BTreeMap::default(),
880            log_namespaces: BTreeSet::new(),
881        };
882
883        let mut got = Definition::new_with_default_metadata(Kind::object(Collection::empty()), []);
884        got = got.unknown_fields(Kind::boolean());
885        got = got.unknown_fields(Kind::bytes().or_integer());
886
887        assert_eq!(got, want);
888    }
889
890    #[test]
891    fn test_meaning_path() {
892        let def = Definition::new(
893            Kind::object(Collection::empty()),
894            Kind::object(Collection::empty()),
895            [LogNamespace::Legacy],
896        )
897        .with_event_field(
898            &owned_value_path!("foo"),
899            Kind::boolean(),
900            Some("foo_meaning"),
901        )
902        .with_metadata_field(
903            &owned_value_path!("bar"),
904            Kind::boolean(),
905            Some("bar_meaning"),
906        );
907
908        assert_eq!(
909            def.meaning_path("foo_meaning").unwrap(),
910            &OwnedTargetPath::event(owned_value_path!("foo"))
911        );
912        assert_eq!(
913            def.meaning_path("bar_meaning").unwrap(),
914            &OwnedTargetPath::metadata(owned_value_path!("bar"))
915        );
916    }
917
918    #[test]
919    #[allow(clippy::too_many_lines)]
920    fn test_merge() {
921        struct TestCase {
922            this: Definition,
923            other: Definition,
924            want: Definition,
925        }
926
927        for (title, TestCase { this, other, want }) in HashMap::from([
928            (
929                "equal definitions",
930                TestCase {
931                    this: Definition {
932                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
933                            "foo".into(),
934                            Kind::boolean().or_null(),
935                        )]))),
936                        metadata_kind: Kind::object(Collection::empty()),
937                        meaning: BTreeMap::from([(
938                            "foo_meaning".to_owned(),
939                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
940                        )]),
941                        log_namespaces: BTreeSet::new(),
942                    },
943                    other: Definition {
944                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
945                            "foo".into(),
946                            Kind::boolean().or_null(),
947                        )]))),
948                        metadata_kind: Kind::object(Collection::empty()),
949                        meaning: BTreeMap::from([(
950                            "foo_meaning".to_owned(),
951                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
952                        )]),
953                        log_namespaces: BTreeSet::new(),
954                    },
955                    want: Definition {
956                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
957                            "foo".into(),
958                            Kind::boolean().or_null(),
959                        )]))),
960                        metadata_kind: Kind::object(Collection::empty()),
961                        meaning: BTreeMap::from([(
962                            "foo_meaning".to_owned(),
963                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
964                        )]),
965                        log_namespaces: BTreeSet::new(),
966                    },
967                },
968            ),
969            (
970                "this optional, other required",
971                TestCase {
972                    this: Definition {
973                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
974                            "foo".into(),
975                            Kind::boolean().or_null(),
976                        )]))),
977                        metadata_kind: Kind::object(Collection::empty()),
978                        meaning: BTreeMap::default(),
979                        log_namespaces: BTreeSet::new(),
980                    },
981                    other: Definition {
982                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
983                            "foo".into(),
984                            Kind::boolean(),
985                        )]))),
986                        metadata_kind: Kind::object(Collection::empty()),
987                        meaning: BTreeMap::default(),
988                        log_namespaces: BTreeSet::new(),
989                    },
990                    want: Definition {
991                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
992                            "foo".into(),
993                            Kind::boolean().or_null(),
994                        )]))),
995                        metadata_kind: Kind::object(Collection::empty()),
996                        meaning: BTreeMap::default(),
997                        log_namespaces: BTreeSet::new(),
998                    },
999                },
1000            ),
1001            (
1002                "this required, other optional",
1003                TestCase {
1004                    this: Definition {
1005                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1006                            "foo".into(),
1007                            Kind::boolean(),
1008                        )]))),
1009                        metadata_kind: Kind::object(Collection::empty()),
1010                        meaning: BTreeMap::default(),
1011                        log_namespaces: BTreeSet::new(),
1012                    },
1013                    other: Definition {
1014                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1015                            "foo".into(),
1016                            Kind::boolean().or_null(),
1017                        )]))),
1018                        metadata_kind: Kind::object(Collection::empty()),
1019                        meaning: BTreeMap::default(),
1020                        log_namespaces: BTreeSet::new(),
1021                    },
1022                    want: Definition {
1023                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1024                            "foo".into(),
1025                            Kind::boolean().or_null(),
1026                        )]))),
1027                        metadata_kind: Kind::object(Collection::empty()),
1028                        meaning: BTreeMap::default(),
1029                        log_namespaces: BTreeSet::new(),
1030                    },
1031                },
1032            ),
1033            (
1034                "this required, other required",
1035                TestCase {
1036                    this: Definition {
1037                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1038                            "foo".into(),
1039                            Kind::boolean(),
1040                        )]))),
1041                        metadata_kind: Kind::object(Collection::empty()),
1042                        meaning: BTreeMap::default(),
1043                        log_namespaces: BTreeSet::new(),
1044                    },
1045                    other: Definition {
1046                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1047                            "foo".into(),
1048                            Kind::boolean(),
1049                        )]))),
1050                        metadata_kind: Kind::object(Collection::empty()),
1051                        meaning: BTreeMap::default(),
1052                        log_namespaces: BTreeSet::new(),
1053                    },
1054                    want: Definition {
1055                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1056                            "foo".into(),
1057                            Kind::boolean(),
1058                        )]))),
1059                        metadata_kind: Kind::object(Collection::empty()),
1060                        meaning: BTreeMap::default(),
1061                        log_namespaces: BTreeSet::new(),
1062                    },
1063                },
1064            ),
1065            (
1066                "same meaning, pointing to different paths",
1067                TestCase {
1068                    this: Definition {
1069                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1070                            "foo".into(),
1071                            Kind::boolean(),
1072                        )]))),
1073                        metadata_kind: Kind::object(Collection::empty()),
1074                        meaning: BTreeMap::from([(
1075                            "foo".into(),
1076                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
1077                        )]),
1078                        log_namespaces: BTreeSet::new(),
1079                    },
1080                    other: Definition {
1081                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1082                            "foo".into(),
1083                            Kind::boolean(),
1084                        )]))),
1085                        metadata_kind: Kind::object(Collection::empty()),
1086                        meaning: BTreeMap::from([(
1087                            "foo".into(),
1088                            MeaningPointer::Valid(parse_target_path("bar").unwrap()),
1089                        )]),
1090                        log_namespaces: BTreeSet::new(),
1091                    },
1092                    want: Definition {
1093                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1094                            "foo".into(),
1095                            Kind::boolean(),
1096                        )]))),
1097                        metadata_kind: Kind::object(Collection::empty()),
1098                        meaning: BTreeMap::from([(
1099                            "foo".into(),
1100                            MeaningPointer::Invalid(BTreeSet::from([
1101                                parse_target_path("foo").unwrap(),
1102                                parse_target_path("bar").unwrap(),
1103                            ])),
1104                        )]),
1105                        log_namespaces: BTreeSet::new(),
1106                    },
1107                },
1108            ),
1109            (
1110                "same meaning, pointing to same path",
1111                TestCase {
1112                    this: Definition {
1113                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1114                            "foo".into(),
1115                            Kind::boolean(),
1116                        )]))),
1117                        metadata_kind: Kind::object(Collection::empty()),
1118                        meaning: BTreeMap::from([(
1119                            "foo".into(),
1120                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
1121                        )]),
1122                        log_namespaces: BTreeSet::new(),
1123                    },
1124                    other: Definition {
1125                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1126                            "foo".into(),
1127                            Kind::boolean(),
1128                        )]))),
1129                        metadata_kind: Kind::object(Collection::empty()),
1130                        meaning: BTreeMap::from([(
1131                            "foo".into(),
1132                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
1133                        )]),
1134                        log_namespaces: BTreeSet::new(),
1135                    },
1136                    want: Definition {
1137                        event_kind: Kind::object(Collection::from(BTreeMap::from([(
1138                            "foo".into(),
1139                            Kind::boolean(),
1140                        )]))),
1141                        metadata_kind: Kind::object(Collection::empty()),
1142                        meaning: BTreeMap::from([(
1143                            "foo".into(),
1144                            MeaningPointer::Valid(parse_target_path("foo").unwrap()),
1145                        )]),
1146                        log_namespaces: BTreeSet::new(),
1147                    },
1148                },
1149            ),
1150        ]) {
1151            let got = this.merge(other);
1152
1153            assert_eq!(got, want, "{title}");
1154        }
1155    }
1156}
vector_core/schema/definition.rs

vector_core/schema/
definition.rs
