vector/enrichment_tables/
mod.rs

1//! Functionality to handle enrichment tables.
2use std::path::PathBuf;
3
4use enum_dispatch::enum_dispatch;
5use vector_lib::configurable::configurable_component;
6pub use vector_lib::enrichment::{Condition, IndexHandle, Table};
7
8use crate::config::{
9    ComponentKey, EnrichmentTableConfig, GenerateConfig, GlobalOptions, SinkConfig, SourceConfig,
10};
11
12pub mod file;
13
14#[cfg(feature = "enrichment-tables-memory")]
15pub mod memory;
16
17#[cfg(feature = "enrichment-tables-geoip")]
18pub mod geoip;
19
20#[cfg(feature = "enrichment-tables-mmdb")]
21pub mod mmdb;
22
23/// Configuration options for an [enrichment table](https://vector.dev/docs/reference/glossary/#enrichment-tables) to be used in a
24/// [`remap`](https://vector.dev/docs/reference/configuration/transforms/remap/) transform. Currently supported are:
25///
26/// * [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) files
27/// * [MaxMind](https://www.maxmind.com/en/home) databases
28/// * In-memory storage
29///
30/// For the lookup in the enrichment tables to be as performant as possible, the data is indexed according
31/// to the fields that are used in the search. Note that indices can only be created for fields for which an
32/// exact match is used in the condition. For range searches, an index isn't used and the enrichment table
33/// drops back to a sequential scan of the data. A sequential scan shouldn't impact performance
34/// significantly provided that there are only a few possible rows returned by the exact matches in the
35/// condition. We don't recommend using a condition that uses only date range searches.
36///
37///
38#[configurable_component]
39#[derive(Clone, Debug)]
40#[serde(tag = "type", rename_all = "snake_case")]
41#[enum_dispatch(EnrichmentTableConfig)]
42#[configurable(metadata(
43    docs::enum_tag_description = "enrichment table type",
44    docs::common = false,
45    docs::required = false,
46))]
47pub enum EnrichmentTables {
48    /// Exposes data from a static file as an enrichment table.
49    File(file::FileConfig),
50
51    /// Exposes data from a memory cache as an enrichment table. The cache can be written to using
52    /// a sink.
53    #[cfg(feature = "enrichment-tables-memory")]
54    Memory(memory::MemoryConfig),
55
56    /// Exposes data from a [MaxMind][maxmind] [GeoIP2][geoip2] database as an enrichment table.
57    ///
58    /// [maxmind]: https://www.maxmind.com/
59    /// [geoip2]: https://www.maxmind.com/en/geoip2-databases
60    #[cfg(feature = "enrichment-tables-geoip")]
61    Geoip(geoip::GeoipConfig),
62
63    /// Exposes data from a [MaxMind][maxmind] database as an enrichment table.
64    ///
65    /// [maxmind]: https://www.maxmind.com/
66    #[cfg(feature = "enrichment-tables-mmdb")]
67    Mmdb(mmdb::MmdbConfig),
68}
69
70// Manual NamedComponent impl required because enum_dispatch doesn't support it yet.
71impl vector_lib::configurable::NamedComponent for EnrichmentTables {
72    fn get_component_name(&self) -> &'static str {
73        match self {
74            Self::File(config) => config.get_component_name(),
75            #[cfg(feature = "enrichment-tables-memory")]
76            Self::Memory(config) => config.get_component_name(),
77            #[cfg(feature = "enrichment-tables-geoip")]
78            Self::Geoip(config) => config.get_component_name(),
79            #[cfg(feature = "enrichment-tables-mmdb")]
80            Self::Mmdb(config) => config.get_component_name(),
81        }
82    }
83}
84
85impl GenerateConfig for EnrichmentTables {
86    fn generate_config() -> toml::Value {
87        toml::Value::try_from(Self::File(file::FileConfig {
88            file: file::FileSettings {
89                path: "path/to/file".into(),
90                encoding: file::Encoding::default(),
91            },
92            schema: Default::default(),
93        }))
94        .unwrap()
95    }
96}
97
98impl EnrichmentTables {
99    /// Gets the files to watch to trigger reload
100    pub fn files_to_watch(&self) -> Vec<&PathBuf> {
101        match self {
102            EnrichmentTables::File(file_config) => vec![&file_config.file.path],
103            #[cfg(feature = "enrichment-tables-memory")]
104            EnrichmentTables::Memory(_) => vec![],
105            #[cfg(feature = "enrichment-tables-geoip")]
106            EnrichmentTables::Geoip(geoip_config) => vec![&geoip_config.path],
107            #[cfg(feature = "enrichment-tables-mmdb")]
108            EnrichmentTables::Mmdb(mmdb_config) => vec![&mmdb_config.path],
109        }
110    }
111}
vector/enrichment_tables/mod.rs

vector/enrichment_tables/
mod.rs
