vector/config/loading/
config_builder.rs

1use std::{collections::HashMap, io::Read};
2
3use indexmap::IndexMap;
4use toml::value::Table;
5
6use super::{ComponentHint, Process, deserialize_table, loader, prepare_input, secret};
7use crate::config::{
8    ComponentKey, ConfigBuilder, EnrichmentTableOuter, SinkOuter, SourceOuter, TestDefinition,
9    TransformOuter,
10};
11
12#[derive(Debug)]
13pub struct ConfigBuilderLoader {
14    builder: ConfigBuilder,
15    secrets: HashMap<String, String>,
16    interpolate_env: bool,
17}
18
19impl ConfigBuilderLoader {
20    /// Sets whether to interpolate environment variables in the config.
21    pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
22        self.interpolate_env = interpolate;
23        self
24    }
25
26    /// Sets the secrets map for secret interpolation.
27    pub fn secrets(mut self, secrets: HashMap<String, String>) -> Self {
28        self.secrets = secrets;
29        self
30    }
31
32    /// Sets whether to allow empty configuration.
33    pub const fn allow_empty(mut self, allow_empty: bool) -> Self {
34        self.builder.allow_empty = allow_empty;
35        self
36    }
37
38    /// Builds the ConfigBuilderLoader and loads configuration from the specified paths.
39    pub fn load_from_paths(
40        self,
41        config_paths: &[super::ConfigPath],
42    ) -> Result<ConfigBuilder, Vec<String>> {
43        super::loader_from_paths(self, config_paths)
44    }
45
46    /// Builds the ConfigBuilderLoader and loads configuration from an input reader.
47    pub fn load_from_input<R: Read>(
48        self,
49        input: R,
50        format: super::Format,
51    ) -> Result<ConfigBuilder, Vec<String>> {
52        super::loader_from_input(self, input, format)
53    }
54}
55
56impl Default for ConfigBuilderLoader {
57    /// Creates a new builder with default settings.
58    /// By default, environment variable interpolation is enabled.
59    fn default() -> Self {
60        Self {
61            builder: ConfigBuilder::default(),
62            secrets: HashMap::new(),
63            interpolate_env: true,
64        }
65    }
66}
67
68impl Process for ConfigBuilderLoader {
69    /// Prepares input for a `ConfigBuilder` by interpolating environment variables.
70    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
71        let prepared_input = prepare_input(input, self.interpolate_env)?;
72        Ok(if self.secrets.is_empty() {
73            prepared_input
74        } else {
75            secret::interpolate(&prepared_input, &self.secrets)?
76        })
77    }
78
79    /// Merge a TOML `Table` with a `ConfigBuilder`. Component types extend specific keys.
80    fn merge(&mut self, table: Table, hint: Option<ComponentHint>) -> Result<(), Vec<String>> {
81        match hint {
82            Some(ComponentHint::Source) => {
83                self.builder.sources.extend(deserialize_table::<
84                    IndexMap<ComponentKey, SourceOuter>,
85                >(table)?);
86            }
87            Some(ComponentHint::Sink) => {
88                self.builder.sinks.extend(
89                    deserialize_table::<IndexMap<ComponentKey, SinkOuter<_>>>(table)?,
90                );
91            }
92            Some(ComponentHint::Transform) => {
93                self.builder.transforms.extend(deserialize_table::<
94                    IndexMap<ComponentKey, TransformOuter<_>>,
95                >(table)?);
96            }
97            Some(ComponentHint::EnrichmentTable) => {
98                self.builder.enrichment_tables.extend(deserialize_table::<
99                    IndexMap<ComponentKey, EnrichmentTableOuter<_>>,
100                >(table)?);
101            }
102            Some(ComponentHint::Test) => {
103                // This serializes to a `Vec<TestDefinition<_>>`, so we need to first expand
104                // it to an ordered map, and then pull out the value, ignoring the keys.
105                self.builder.tests.extend(
106                    deserialize_table::<IndexMap<String, TestDefinition<String>>>(table)?
107                        .into_iter()
108                        .map(|(_, test)| test),
109                );
110            }
111            None => {
112                self.builder.append(deserialize_table(table)?)?;
113            }
114        };
115
116        Ok(())
117    }
118}
119
120impl loader::Loader<ConfigBuilder> for ConfigBuilderLoader {
121    /// Returns the resulting `ConfigBuilder`.
122    fn take(self) -> ConfigBuilder {
123        self.builder
124    }
125}
126
127#[cfg(all(
128    test,
129    feature = "sinks-elasticsearch",
130    feature = "transforms-sample",
131    feature = "sources-demo_logs",
132    feature = "sinks-console"
133))]
134mod tests {
135    use std::path::PathBuf;
136
137    use super::ConfigBuilderLoader;
138    use crate::config::{ComponentKey, ConfigPath};
139
140    #[test]
141    fn load_namespacing_folder() {
142        let path = PathBuf::from(".")
143            .join("tests")
144            .join("namespacing")
145            .join("success");
146        let configs = vec![ConfigPath::Dir(path)];
147        let builder = ConfigBuilderLoader::default()
148            .interpolate_env(true)
149            .load_from_paths(&configs)
150            .unwrap();
151        assert!(
152            builder
153                .transforms
154                .contains_key(&ComponentKey::from("apache_parser"))
155        );
156        assert!(
157            builder
158                .sources
159                .contains_key(&ComponentKey::from("apache_logs"))
160        );
161        assert!(
162            builder
163                .sinks
164                .contains_key(&ComponentKey::from("es_cluster"))
165        );
166        assert_eq!(builder.tests.len(), 2);
167    }
168
169    #[test]
170    fn load_namespacing_ignore_invalid() {
171        let path = PathBuf::from(".")
172            .join("tests")
173            .join("namespacing")
174            .join("ignore-invalid");
175        let configs = vec![ConfigPath::Dir(path)];
176        ConfigBuilderLoader::default()
177            .interpolate_env(true)
178            .load_from_paths(&configs)
179            .unwrap();
180    }
181
182    #[test]
183    fn load_directory_ignores_unknown_file_formats() {
184        let path = PathBuf::from(".")
185            .join("tests")
186            .join("config-dir")
187            .join("ignore-unknown");
188        let configs = vec![ConfigPath::Dir(path)];
189        ConfigBuilderLoader::default()
190            .interpolate_env(true)
191            .load_from_paths(&configs)
192            .unwrap();
193    }
194
195    #[test]
196    fn load_directory_globals() {
197        let path = PathBuf::from(".")
198            .join("tests")
199            .join("config-dir")
200            .join("globals");
201        let configs = vec![ConfigPath::Dir(path)];
202        ConfigBuilderLoader::default()
203            .interpolate_env(true)
204            .load_from_paths(&configs)
205            .unwrap();
206    }
207
208    #[test]
209    fn load_directory_globals_duplicates() {
210        let path = PathBuf::from(".")
211            .join("tests")
212            .join("config-dir")
213            .join("globals-duplicate");
214        let configs = vec![ConfigPath::Dir(path)];
215        ConfigBuilderLoader::default()
216            .interpolate_env(true)
217            .load_from_paths(&configs)
218            .unwrap();
219    }
220}
vector/config/loading/config_builder.rs

vector/config/loading/
config_builder.rs
