vector/config/loading/
secret.rs

1use std::{
2    collections::{HashMap, HashSet},
3    io::Read,
4    sync::LazyLock,
5};
6
7use futures::TryFutureExt;
8use indexmap::IndexMap;
9use regex::{Captures, Regex};
10use serde::{Deserialize, Serialize};
11use toml::value::Table;
12use vector_lib::config::ComponentKey;
13
14use crate::{
15    config::{
16        SecretBackend,
17        loading::{ComponentHint, Loader, deserialize_table, prepare_input, process::Process},
18    },
19    secrets::SecretBackends,
20    signal,
21};
22
23// The following regex aims to extract a pair of strings, the first being the secret backend name
24// and the second being the secret key. Here are some matching & non-matching examples:
25// - "SECRET[backend.secret_name]" will match and capture "backend" and "secret_name"
26// - "SECRET[backend.secret.name]" will match and capture "backend" and "secret.name"
27// - "SECRET[backend..secret.name]" will match and capture "backend" and ".secret.name"
28// - "SECRET[secret_name]" will not match
29// - "SECRET[.secret.name]" will not match
30pub static COLLECTOR: LazyLock<Regex> =
31    LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].-]+)\]").unwrap());
32
33/// Helper type for specifically deserializing secrets backends.
34#[derive(Debug, Default, Deserialize, Serialize)]
35pub(crate) struct SecretBackendOuter {
36    #[serde(default)]
37    pub(crate) secret: IndexMap<ComponentKey, SecretBackends>,
38}
39
40/// Loader for secrets backends.
41#[derive(Debug, Deserialize, Serialize)]
42pub struct SecretBackendLoader {
43    backends: IndexMap<ComponentKey, SecretBackends>,
44    secret_keys: HashMap<String, HashSet<String>>,
45    interpolate_env: bool,
46}
47
48impl SecretBackendLoader {
49    /// Sets whether to interpolate environment variables in the config.
50    pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
51        self.interpolate_env = interpolate;
52        self
53    }
54
55    /// Retrieve secrets from backends.
56    /// Returns an empty HashMap if there are no secrets to retrieve.
57    pub(crate) async fn retrieve_secrets(
58        mut self,
59        signal_handler: &mut signal::SignalHandler,
60    ) -> Result<HashMap<String, String>, String> {
61        if self.secret_keys.is_empty() {
62            debug!(message = "No secret placeholder found, skipping secret resolution.");
63            return Ok(HashMap::new());
64        }
65
66        debug!(message = "Secret placeholders found, retrieving secrets from configured backends.");
67        let mut secrets: HashMap<String, String> = HashMap::new();
68        let mut signal_rx = signal_handler.subscribe();
69
70        for (backend_name, keys) in &self.secret_keys {
71            let backend = self
72                .backends
73                .get_mut(&ComponentKey::from(backend_name.clone()))
74                .ok_or_else(|| {
75                    format!(
76                        "Backend \"{backend_name}\" is required for secret retrieval but was not found in config."
77                    )
78                })?;
79
80            debug!(message = "Retrieving secrets from a backend.", backend = ?backend_name, keys = ?keys);
81            let backend_secrets = backend
82                .retrieve(keys.clone(), &mut signal_rx)
83                .map_err(|e| {
84                    format!("Error while retrieving secret from backend \"{backend_name}\": {e}.")
85                })
86                .await?;
87
88            for (k, v) in backend_secrets {
89                trace!(message = "Successfully retrieved a secret.", backend = ?backend_name, key = ?k);
90                secrets.insert(format!("{backend_name}.{k}"), v);
91            }
92        }
93
94        Ok(secrets)
95    }
96}
97
98impl Default for SecretBackendLoader {
99    /// Creates a new SecretBackendLoader with default settings.
100    /// By default, environment variable interpolation is enabled.
101    fn default() -> Self {
102        Self {
103            backends: IndexMap::new(),
104            secret_keys: HashMap::new(),
105            interpolate_env: true,
106        }
107    }
108}
109
110impl Process for SecretBackendLoader {
111    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
112        let config_string = prepare_input(input, self.interpolate_env)?;
113        // Collect secret placeholders just after env var processing
114        collect_secret_keys(&config_string, &mut self.secret_keys);
115        Ok(config_string)
116    }
117
118    fn merge(&mut self, table: Table, _: Option<ComponentHint>) -> Result<(), Vec<String>> {
119        if table.contains_key("secret") {
120            let additional = deserialize_table::<SecretBackendOuter>(table)?;
121            self.backends.extend(additional.secret);
122        }
123        Ok(())
124    }
125}
126
127impl Loader<SecretBackendLoader> for SecretBackendLoader {
128    fn take(self) -> SecretBackendLoader {
129        self
130    }
131}
132
133fn collect_secret_keys(input: &str, keys: &mut HashMap<String, HashSet<String>>) {
134    COLLECTOR.captures_iter(input).for_each(|cap| {
135        if let (Some(backend), Some(key)) = (cap.get(1), cap.get(2)) {
136            if let Some(keys) = keys.get_mut(backend.as_str()) {
137                keys.insert(key.as_str().to_string());
138            } else {
139                keys.insert(
140                    backend.as_str().to_string(),
141                    HashSet::from_iter(std::iter::once(key.as_str().to_string())),
142                );
143            }
144        }
145    });
146}
147
148pub fn interpolate(input: &str, secrets: &HashMap<String, String>) -> Result<String, Vec<String>> {
149    let mut errors = Vec::<String>::new();
150    let output = COLLECTOR
151        .replace_all(input, |caps: &Captures<'_>| {
152            caps.get(1)
153                .and_then(|b| caps.get(2).map(|k| (b, k)))
154                .and_then(|(b, k)| secrets.get(&format!("{}.{}", b.as_str(), k.as_str())))
155                .cloned()
156                .unwrap_or_else(|| {
157                    errors.push(format!(
158                        "Unable to find secret replacement for {}.",
159                        caps.get(0).unwrap().as_str()
160                    ));
161                    "".to_string()
162                })
163        })
164        .into_owned();
165    if errors.is_empty() {
166        Ok(output)
167    } else {
168        Err(errors)
169    }
170}
171
172#[cfg(test)]
173mod tests {
174    use std::collections::HashMap;
175
176    use indoc::indoc;
177
178    use super::{collect_secret_keys, interpolate};
179
180    #[test]
181    fn replacement() {
182        let secrets: HashMap<String, String> = vec![
183            ("a.secret.key".into(), "value".into()),
184            ("a...key".into(), "a...value".into()),
185        ]
186        .into_iter()
187        .collect();
188
189        assert_eq!(
190            Ok("value".into()),
191            interpolate("SECRET[a.secret.key]", &secrets)
192        );
193        assert_eq!(
194            Ok("value value".into()),
195            interpolate("SECRET[a.secret.key] SECRET[a.secret.key]", &secrets)
196        );
197
198        assert_eq!(
199            Ok("xxxvalueyyy".into()),
200            interpolate("xxxSECRET[a.secret.key]yyy", &secrets)
201        );
202        assert_eq!(
203            Ok("a...value".into()),
204            interpolate("SECRET[a...key]", &secrets)
205        );
206        assert_eq!(
207            Ok("xxxSECRET[non_matching_syntax]yyy".into()),
208            interpolate("xxxSECRET[non_matching_syntax]yyy", &secrets)
209        );
210        assert_eq!(
211            Err(vec![
212                "Unable to find secret replacement for SECRET[a.non.existing.key].".into()
213            ]),
214            interpolate("xxxSECRET[a.non.existing.key]yyy", &secrets)
215        );
216    }
217
218    #[test]
219    fn collection() {
220        let mut keys = HashMap::new();
221        collect_secret_keys(
222            indoc! {r"
223            SECRET[first_backend.secret_key]
224            SECRET[first_backend.secret-key]
225            SECRET[first_backend.another_secret_key]
226            SECRET[second_backend.secret_key]
227            SECRET[second_backend.secret.key]
228            SECRET[first_backend.a_third.secret_key]
229            SECRET[first_backend...an_extra_secret_key]
230            SECRET[non_matching_syntax]
231            SECRET[.non.matching.syntax]
232        "},
233            &mut keys,
234        );
235        assert_eq!(keys.len(), 2);
236        assert!(keys.contains_key("first_backend"));
237        assert!(keys.contains_key("second_backend"));
238
239        let first_backend_keys = keys.get("first_backend").unwrap();
240        assert_eq!(first_backend_keys.len(), 5);
241        assert!(first_backend_keys.contains("secret_key"));
242        assert!(first_backend_keys.contains("secret-key"));
243        assert!(first_backend_keys.contains("another_secret_key"));
244        assert!(first_backend_keys.contains("a_third.secret_key"));
245        assert!(first_backend_keys.contains("..an_extra_secret_key"));
246
247        let second_backend_keys = keys.get("second_backend").unwrap();
248        assert_eq!(second_backend_keys.len(), 2);
249        assert!(second_backend_keys.contains("secret_key"));
250        assert!(second_backend_keys.contains("secret.key"));
251    }
252
253    #[test]
254    fn collection_duplicates() {
255        let mut keys = HashMap::new();
256        collect_secret_keys(
257            indoc! {r"
258            SECRET[first_backend.secret_key]
259            SECRET[first_backend.secret_key]
260        "},
261            &mut keys,
262        );
263
264        let first_backend_keys = keys.get("first_backend").unwrap();
265        assert_eq!(first_backend_keys.len(), 1);
266        assert!(first_backend_keys.contains("secret_key"));
267    }
268}
vector/config/loading/secret.rs

vector/config/loading/
secret.rs
