vector/config/loading/
secret.rs

1use std::{
2    collections::{HashMap, HashSet},
3    io::Read,
4    sync::LazyLock,
5};
6
7use futures::TryFutureExt;
8use indexmap::IndexMap;
9use regex::{Captures, Regex};
10use serde::{Deserialize, Serialize};
11use toml::value::Table;
12use vector_lib::config::ComponentKey;
13
14use crate::{
15    config::{
16        SecretBackend,
17        loading::{ComponentHint, Loader, deserialize_table, prepare_input, process::Process},
18    },
19    secrets::SecretBackends,
20    signal,
21};
22
23// The following regex aims to extract a pair of strings, the first being the secret backend name
24// and the second being the secret key. Here are some matching & non-matching examples:
25// - "SECRET[backend.secret_name]" will match and capture "backend" and "secret_name"
26// - "SECRET[backend.secret.name]" will match and capture "backend" and "secret.name"
27// - "SECRET[backend..secret.name]" will match and capture "backend" and ".secret.name"
28// - "SECRET[backend.path/to/secret]" will match and capture "backend" and "path/to/secret"
29// - "SECRET[secret_name]" will not match
30// - "SECRET[.secret.name]" will not match
31pub static COLLECTOR: LazyLock<Regex> =
32    LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].\-/]+)\]").unwrap());
33
34/// Helper type for specifically deserializing secrets backends.
35#[derive(Debug, Default, Deserialize, Serialize)]
36pub(crate) struct SecretBackendOuter {
37    #[serde(default)]
38    pub(crate) secret: IndexMap<ComponentKey, SecretBackends>,
39}
40
41/// Loader for secrets backends.
42#[derive(Debug, Deserialize, Serialize)]
43pub struct SecretBackendLoader {
44    backends: IndexMap<ComponentKey, SecretBackends>,
45    secret_keys: HashMap<String, HashSet<String>>,
46    interpolate_env: bool,
47}
48
49impl SecretBackendLoader {
50    /// Sets whether to interpolate environment variables in the config.
51    pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
52        self.interpolate_env = interpolate;
53        self
54    }
55
56    /// Retrieve secrets from backends.
57    /// Returns an empty HashMap if there are no secrets to retrieve.
58    pub(crate) async fn retrieve_secrets(
59        mut self,
60        signal_handler: &mut signal::SignalHandler,
61    ) -> Result<HashMap<String, String>, String> {
62        if self.secret_keys.is_empty() {
63            debug!(message = "No secret placeholder found, skipping secret resolution.");
64            return Ok(HashMap::new());
65        }
66
67        debug!(message = "Secret placeholders found, retrieving secrets from configured backends.");
68        let mut secrets: HashMap<String, String> = HashMap::new();
69        let mut signal_rx = signal_handler.subscribe();
70
71        for (backend_name, keys) in &self.secret_keys {
72            let backend = self
73                .backends
74                .get_mut(&ComponentKey::from(backend_name.clone()))
75                .ok_or_else(|| {
76                    format!(
77                        "Backend \"{backend_name}\" is required for secret retrieval but was not found in config."
78                    )
79                })?;
80
81            debug!(message = "Retrieving secrets from a backend.", backend = ?backend_name, keys = ?keys);
82            let backend_secrets = backend
83                .retrieve(keys.clone(), &mut signal_rx)
84                .map_err(|e| {
85                    format!("Error while retrieving secret from backend \"{backend_name}\": {e}.")
86                })
87                .await?;
88
89            for (k, v) in backend_secrets {
90                trace!(message = "Successfully retrieved a secret.", backend = ?backend_name, key = ?k);
91                secrets.insert(format!("{backend_name}.{k}"), v);
92            }
93        }
94
95        Ok(secrets)
96    }
97}
98
99impl Default for SecretBackendLoader {
100    fn default() -> Self {
101        Self {
102            backends: IndexMap::new(),
103            secret_keys: HashMap::new(),
104            interpolate_env: super::env_var_interpolation_enabled(),
105        }
106    }
107}
108
109impl Process for SecretBackendLoader {
110    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
111        let config_string = prepare_input(input, self.interpolate_env)?;
112        // Collect secret placeholders just after env var processing
113        collect_secret_keys(&config_string, &mut self.secret_keys);
114        Ok(config_string)
115    }
116
117    fn merge(&mut self, table: Table, _: Option<ComponentHint>) -> Result<(), Vec<String>> {
118        if table.contains_key("secret") {
119            let additional = deserialize_table::<SecretBackendOuter>(table)?;
120            self.backends.extend(additional.secret);
121        }
122        Ok(())
123    }
124}
125
126impl Loader<SecretBackendLoader> for SecretBackendLoader {
127    fn take(self) -> SecretBackendLoader {
128        self
129    }
130}
131
132fn collect_secret_keys(input: &str, keys: &mut HashMap<String, HashSet<String>>) {
133    COLLECTOR.captures_iter(input).for_each(|cap| {
134        if let (Some(backend), Some(key)) = (cap.get(1), cap.get(2)) {
135            if let Some(keys) = keys.get_mut(backend.as_str()) {
136                keys.insert(key.as_str().to_string());
137            } else {
138                keys.insert(
139                    backend.as_str().to_string(),
140                    HashSet::from_iter(std::iter::once(key.as_str().to_string())),
141                );
142            }
143        }
144    });
145}
146
147pub fn interpolate(input: &str, secrets: &HashMap<String, String>) -> Result<String, Vec<String>> {
148    let mut errors = Vec::<String>::new();
149    let output = COLLECTOR
150        .replace_all(input, |caps: &Captures<'_>| {
151            caps.get(1)
152                .and_then(|b| caps.get(2).map(|k| (b, k)))
153                .and_then(|(b, k)| secrets.get(&format!("{}.{}", b.as_str(), k.as_str())))
154                .cloned()
155                .unwrap_or_else(|| {
156                    errors.push(format!(
157                        "Unable to find secret replacement for {}.",
158                        caps.get(0).unwrap().as_str()
159                    ));
160                    "".to_string()
161                })
162        })
163        .into_owned();
164    if errors.is_empty() {
165        Ok(output)
166    } else {
167        Err(errors)
168    }
169}
170
171#[cfg(test)]
172mod tests {
173    use std::collections::HashMap;
174
175    use indoc::indoc;
176
177    use super::{collect_secret_keys, interpolate};
178
179    #[test]
180    fn replacement() {
181        let secrets: HashMap<String, String> = vec![
182            ("a.secret.key".into(), "value".into()),
183            ("a...key".into(), "a...value".into()),
184            ("backend.path/to/secret".into(), "secret_value".into()),
185            ("backend.nested/dir/file".into(), "nested_value".into()),
186        ]
187        .into_iter()
188        .collect();
189
190        assert_eq!(
191            Ok("value".into()),
192            interpolate("SECRET[a.secret.key]", &secrets)
193        );
194        assert_eq!(
195            Ok("value value".into()),
196            interpolate("SECRET[a.secret.key] SECRET[a.secret.key]", &secrets)
197        );
198
199        assert_eq!(
200            Ok("xxxvalueyyy".into()),
201            interpolate("xxxSECRET[a.secret.key]yyy", &secrets)
202        );
203        assert_eq!(
204            Ok("a...value".into()),
205            interpolate("SECRET[a...key]", &secrets)
206        );
207        assert_eq!(
208            Ok("secret_value".into()),
209            interpolate("SECRET[backend.path/to/secret]", &secrets)
210        );
211        assert_eq!(
212            Ok("nested_value".into()),
213            interpolate("SECRET[backend.nested/dir/file]", &secrets)
214        );
215        assert_eq!(
216            Ok("xxxSECRET[non_matching_syntax]yyy".into()),
217            interpolate("xxxSECRET[non_matching_syntax]yyy", &secrets)
218        );
219        assert_eq!(
220            Err(vec![
221                "Unable to find secret replacement for SECRET[a.non.existing.key].".into()
222            ]),
223            interpolate("xxxSECRET[a.non.existing.key]yyy", &secrets)
224        );
225    }
226
227    #[test]
228    fn collection() {
229        let mut keys = HashMap::new();
230        collect_secret_keys(
231            indoc! {r"
232            SECRET[first_backend.secret_key]
233            SECRET[first_backend.secret-key]
234            SECRET[first_backend.another_secret_key]
235            SECRET[second_backend.secret_key]
236            SECRET[second_backend.secret.key]
237            SECRET[first_backend.a_third.secret_key]
238            SECRET[first_backend...an_extra_secret_key]
239            SECRET[first_backend.path/to/secret]
240            SECRET[second_backend.nested/dir/secret]
241            SECRET[non_matching_syntax]
242            SECRET[.non.matching.syntax]
243        "},
244            &mut keys,
245        );
246        assert_eq!(keys.len(), 2);
247        assert!(keys.contains_key("first_backend"));
248        assert!(keys.contains_key("second_backend"));
249
250        let first_backend_keys = keys.get("first_backend").unwrap();
251        assert_eq!(first_backend_keys.len(), 6);
252        assert!(first_backend_keys.contains("secret_key"));
253        assert!(first_backend_keys.contains("secret-key"));
254        assert!(first_backend_keys.contains("another_secret_key"));
255        assert!(first_backend_keys.contains("a_third.secret_key"));
256        assert!(first_backend_keys.contains("..an_extra_secret_key"));
257        assert!(first_backend_keys.contains("path/to/secret"));
258
259        let second_backend_keys = keys.get("second_backend").unwrap();
260        assert_eq!(second_backend_keys.len(), 3);
261        assert!(second_backend_keys.contains("secret_key"));
262        assert!(second_backend_keys.contains("secret.key"));
263        assert!(second_backend_keys.contains("nested/dir/secret"));
264    }
265
266    #[test]
267    fn collection_duplicates() {
268        let mut keys = HashMap::new();
269        collect_secret_keys(
270            indoc! {r"
271            SECRET[first_backend.secret_key]
272            SECRET[first_backend.secret_key]
273        "},
274            &mut keys,
275        );
276
277        let first_backend_keys = keys.get("first_backend").unwrap();
278        assert_eq!(first_backend_keys.len(), 1);
279        assert!(first_backend_keys.contains("secret_key"));
280    }
281}
vector/config/loading/secret.rs

vector/config/loading/
secret.rs
