vector/enrichment_tables/
mod.rs

1//! Functionality to handle enrichment tables.
2use std::path::PathBuf;
3
4use enum_dispatch::enum_dispatch;
5use vector_lib::configurable::configurable_component;
6pub use vector_lib::enrichment::{Condition, IndexHandle, Table};
7
8use crate::config::{
9    ComponentKey, EnrichmentTableConfig, GenerateConfig, GlobalOptions, SinkConfig, SourceConfig,
10};
11
12pub mod file;
13
14#[cfg(feature = "enrichment-tables-memory")]
15pub mod memory;
16
17#[cfg(feature = "enrichment-tables-geoip")]
18pub mod geoip;
19
20#[cfg(feature = "enrichment-tables-mmdb")]
21pub mod mmdb;
22
23/// Configuration options for an [enrichment table](https://vector.dev/docs/reference/glossary/#enrichment-tables) to be used in a
24/// [`remap`](https://vector.dev/docs/reference/configuration/transforms/remap/) transform. Currently supported are:
25///
26/// * [CSV](https://en.wikipedia.org/wiki/Comma-separated_values) files
27/// * [MaxMind](https://www.maxmind.com/en/home) databases
28/// * In-memory storage
29///
30/// For the lookup in the enrichment tables to be as performant as possible, the data is indexed according
31/// to the fields that are used in the search. Note that indices can only be created for fields for which an
32/// exact match is used in the condition. For range searches, an index isn't used and the enrichment table
33/// drops back to a sequential scan of the data. A sequential scan shouldn't impact performance
34/// significantly provided that there are only a few possible rows returned by the exact matches in the
35/// condition. We don't recommend using a condition that uses only date range searches.
36///
37///
38#[configurable_component(global_option("enrichment_tables"))]
39#[derive(Clone, Debug)]
40#[serde(tag = "type", rename_all = "snake_case")]
41#[enum_dispatch(EnrichmentTableConfig)]
42#[configurable(metadata(
43    docs::enum_tag_description = "enrichment table type",
44    docs::common = false,
45    docs::required = false,
46))]
47pub enum EnrichmentTables {
48    /// Exposes data from a static file as an enrichment table.
49    File(file::FileConfig),
50
51    /// Exposes data from a memory cache as an enrichment table. The cache can be written to using
52    /// a sink.
53    #[cfg(feature = "enrichment-tables-memory")]
54    Memory(memory::MemoryConfig),
55
56    /// Exposes data from a [MaxMind][maxmind] [GeoIP2][geoip2] database as an enrichment table.
57    ///
58    /// [maxmind]: https://www.maxmind.com/
59    /// [geoip2]: https://www.maxmind.com/en/geoip2-databases
60    #[cfg(feature = "enrichment-tables-geoip")]
61    Geoip(geoip::GeoipConfig),
62
63    /// Exposes data from a [MaxMind][maxmind] database as an enrichment table.
64    ///
65    /// [maxmind]: https://www.maxmind.com/
66    #[cfg(feature = "enrichment-tables-mmdb")]
67    Mmdb(mmdb::MmdbConfig),
68}
69
70impl GenerateConfig for EnrichmentTables {
71    fn generate_config() -> toml::Value {
72        toml::Value::try_from(Self::File(file::FileConfig {
73            file: file::FileSettings {
74                path: "path/to/file".into(),
75                encoding: file::Encoding::default(),
76            },
77            schema: Default::default(),
78        }))
79        .unwrap()
80    }
81}
82
83impl EnrichmentTables {
84    /// Gets the files to watch to trigger reload
85    pub fn files_to_watch(&self) -> Vec<&PathBuf> {
86        match self {
87            EnrichmentTables::File(file_config) => vec![&file_config.file.path],
88            #[cfg(feature = "enrichment-tables-memory")]
89            EnrichmentTables::Memory(_) => vec![],
90            #[cfg(feature = "enrichment-tables-geoip")]
91            EnrichmentTables::Geoip(geoip_config) => vec![&geoip_config.path],
92            #[cfg(feature = "enrichment-tables-mmdb")]
93            EnrichmentTables::Mmdb(mmdb_config) => vec![&mmdb_config.path],
94        }
95    }
96}
vector/enrichment_tables/mod.rs

vector/enrichment_tables/
mod.rs
