vector/config/loading/
config_builder.rs

1use std::{collections::HashMap, io::Read};
2
3use indexmap::IndexMap;
4use toml::value::Table;
5
6use super::{ComponentHint, Process, deserialize_table, loader, prepare_input, secret};
7use crate::config::{
8    ComponentKey, ConfigBuilder, EnrichmentTableOuter, SinkOuter, SourceOuter, TestDefinition,
9    TransformOuter,
10};
11
12#[derive(Debug)]
13pub struct ConfigBuilderLoader {
14    builder: ConfigBuilder,
15    secrets: HashMap<String, String>,
16    interpolate_env: bool,
17}
18
19impl ConfigBuilderLoader {
20    /// Sets whether to interpolate environment variables in the config.
21    pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
22        self.interpolate_env = interpolate;
23        self
24    }
25
26    /// Sets the secrets map for secret interpolation.
27    pub fn secrets(mut self, secrets: HashMap<String, String>) -> Self {
28        self.secrets = secrets;
29        self
30    }
31
32    /// Sets whether to allow empty configuration.
33    pub const fn allow_empty(mut self, allow_empty: bool) -> Self {
34        self.builder.allow_empty = allow_empty;
35        self
36    }
37
38    /// Builds the ConfigBuilderLoader and loads configuration from the specified paths.
39    pub fn load_from_paths(
40        self,
41        config_paths: &[super::ConfigPath],
42    ) -> Result<ConfigBuilder, Vec<String>> {
43        super::loader_from_paths(self, config_paths)
44    }
45
46    /// Builds the ConfigBuilderLoader and loads configuration from an input reader.
47    pub fn load_from_input<R: Read>(
48        self,
49        input: R,
50        format: super::Format,
51    ) -> Result<ConfigBuilder, Vec<String>> {
52        super::loader_from_input(self, input, format)
53    }
54}
55
56impl Default for ConfigBuilderLoader {
57    fn default() -> Self {
58        Self {
59            builder: ConfigBuilder::default(),
60            secrets: HashMap::new(),
61            interpolate_env: super::env_var_interpolation_enabled(),
62        }
63    }
64}
65
66impl Process for ConfigBuilderLoader {
67    /// Prepares input for a `ConfigBuilder` by interpolating environment variables.
68    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
69        let prepared_input = prepare_input(input, self.interpolate_env)?;
70        Ok(if self.secrets.is_empty() {
71            prepared_input
72        } else {
73            secret::interpolate(&prepared_input, &self.secrets)?
74        })
75    }
76
77    /// Merge a TOML `Table` with a `ConfigBuilder`. Component types extend specific keys.
78    fn merge(&mut self, table: Table, hint: Option<ComponentHint>) -> Result<(), Vec<String>> {
79        match hint {
80            Some(ComponentHint::Source) => {
81                self.builder.sources.extend(deserialize_table::<
82                    IndexMap<ComponentKey, SourceOuter>,
83                >(table)?);
84            }
85            Some(ComponentHint::Sink) => {
86                self.builder.sinks.extend(
87                    deserialize_table::<IndexMap<ComponentKey, SinkOuter<_>>>(table)?,
88                );
89            }
90            Some(ComponentHint::Transform) => {
91                self.builder.transforms.extend(deserialize_table::<
92                    IndexMap<ComponentKey, TransformOuter<_>>,
93                >(table)?);
94            }
95            Some(ComponentHint::EnrichmentTable) => {
96                self.builder.enrichment_tables.extend(deserialize_table::<
97                    IndexMap<ComponentKey, EnrichmentTableOuter<_>>,
98                >(table)?);
99            }
100            Some(ComponentHint::Test) => {
101                // This serializes to a `Vec<TestDefinition<_>>`, so we need to first expand
102                // it to an ordered map, and then pull out the value, ignoring the keys.
103                self.builder.tests.extend(
104                    deserialize_table::<IndexMap<String, TestDefinition<String>>>(table)?
105                        .into_iter()
106                        .map(|(_, test)| test),
107                );
108            }
109            None => {
110                self.builder.append(deserialize_table(table)?)?;
111            }
112        };
113
114        Ok(())
115    }
116}
117
118impl loader::Loader<ConfigBuilder> for ConfigBuilderLoader {
119    /// Returns the resulting `ConfigBuilder`.
120    fn take(self) -> ConfigBuilder {
121        self.builder
122    }
123}
124
125#[cfg(all(
126    test,
127    feature = "sinks-elasticsearch",
128    feature = "transforms-sample",
129    feature = "sources-demo_logs",
130    feature = "sinks-console"
131))]
132mod tests {
133    use std::path::PathBuf;
134
135    use super::ConfigBuilderLoader;
136    use crate::config::{ComponentKey, ConfigPath};
137
138    #[test]
139    fn load_namespacing_folder() {
140        let path = PathBuf::from(".")
141            .join("tests")
142            .join("namespacing")
143            .join("success");
144        let configs = vec![ConfigPath::Dir(path)];
145        let builder = ConfigBuilderLoader::default()
146            .interpolate_env(true)
147            .load_from_paths(&configs)
148            .unwrap();
149        assert!(
150            builder
151                .transforms
152                .contains_key(&ComponentKey::from("apache_parser"))
153        );
154        assert!(
155            builder
156                .sources
157                .contains_key(&ComponentKey::from("apache_logs"))
158        );
159        assert!(
160            builder
161                .sinks
162                .contains_key(&ComponentKey::from("es_cluster"))
163        );
164        assert_eq!(builder.tests.len(), 2);
165    }
166
167    #[test]
168    fn load_namespacing_ignore_invalid() {
169        let path = PathBuf::from(".")
170            .join("tests")
171            .join("namespacing")
172            .join("ignore-invalid");
173        let configs = vec![ConfigPath::Dir(path)];
174        ConfigBuilderLoader::default()
175            .interpolate_env(true)
176            .load_from_paths(&configs)
177            .unwrap();
178    }
179
180    #[test]
181    fn load_directory_ignores_unknown_file_formats() {
182        let path = PathBuf::from(".")
183            .join("tests")
184            .join("config-dir")
185            .join("ignore-unknown");
186        let configs = vec![ConfigPath::Dir(path)];
187        ConfigBuilderLoader::default()
188            .interpolate_env(true)
189            .load_from_paths(&configs)
190            .unwrap();
191    }
192
193    #[test]
194    fn load_directory_globals() {
195        let path = PathBuf::from(".")
196            .join("tests")
197            .join("config-dir")
198            .join("globals");
199        let configs = vec![ConfigPath::Dir(path)];
200        ConfigBuilderLoader::default()
201            .interpolate_env(true)
202            .load_from_paths(&configs)
203            .unwrap();
204    }
205
206    #[test]
207    fn load_directory_globals_duplicates() {
208        let path = PathBuf::from(".")
209            .join("tests")
210            .join("config-dir")
211            .join("globals-duplicate");
212        let configs = vec![ConfigPath::Dir(path)];
213        ConfigBuilderLoader::default()
214            .interpolate_env(true)
215            .load_from_paths(&configs)
216            .unwrap();
217    }
218}
vector/config/loading/config_builder.rs

vector/config/loading/
config_builder.rs
