vector/config/loading/
secret.rs

1use std::{
2    collections::{HashMap, HashSet},
3    io::Read,
4    sync::LazyLock,
5};
6
7use futures::TryFutureExt;
8use indexmap::IndexMap;
9use regex::{Captures, Regex};
10use serde::{Deserialize, Serialize};
11use toml::value::Table;
12use vector_lib::config::ComponentKey;
13
14use crate::{
15    config::{
16        SecretBackend,
17        loading::{ComponentHint, Loader, deserialize_table, prepare_input, process::Process},
18    },
19    secrets::SecretBackends,
20    signal,
21};
22
23// The following regex aims to extract a pair of strings, the first being the secret backend name
24// and the second being the secret key. Here are some matching & non-matching examples:
25// - "SECRET[backend.secret_name]" will match and capture "backend" and "secret_name"
26// - "SECRET[backend.secret.name]" will match and capture "backend" and "secret.name"
27// - "SECRET[backend..secret.name]" will match and capture "backend" and ".secret.name"
28// - "SECRET[backend.path/to/secret]" will match and capture "backend" and "path/to/secret"
29// - "SECRET[secret_name]" will not match
30// - "SECRET[.secret.name]" will not match
31pub static COLLECTOR: LazyLock<Regex> =
32    LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]]+)\.([[:word:].\-/]+)\]").unwrap());
33
34/// Helper type for specifically deserializing secrets backends.
35#[derive(Debug, Default, Deserialize, Serialize)]
36pub(crate) struct SecretBackendOuter {
37    #[serde(default)]
38    pub(crate) secret: IndexMap<ComponentKey, SecretBackends>,
39}
40
41/// Loader for secrets backends.
42#[derive(Debug, Deserialize, Serialize)]
43pub struct SecretBackendLoader {
44    backends: IndexMap<ComponentKey, SecretBackends>,
45    secret_keys: HashMap<String, HashSet<String>>,
46    interpolate_env: bool,
47}
48
49impl SecretBackendLoader {
50    /// Sets whether to interpolate environment variables in the config.
51    pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
52        self.interpolate_env = interpolate;
53        self
54    }
55
56    /// Retrieve secrets from backends.
57    /// Returns an empty HashMap if there are no secrets to retrieve.
58    pub(crate) async fn retrieve_secrets(
59        mut self,
60        signal_handler: &mut signal::SignalHandler,
61    ) -> Result<HashMap<String, String>, String> {
62        if self.secret_keys.is_empty() {
63            debug!(message = "No secret placeholder found, skipping secret resolution.");
64            return Ok(HashMap::new());
65        }
66
67        debug!(message = "Secret placeholders found, retrieving secrets from configured backends.");
68        let mut secrets: HashMap<String, String> = HashMap::new();
69        let mut signal_rx = signal_handler.subscribe();
70
71        for (backend_name, keys) in &self.secret_keys {
72            let backend = self
73                .backends
74                .get_mut(&ComponentKey::from(backend_name.clone()))
75                .ok_or_else(|| {
76                    format!(
77                        "Backend \"{backend_name}\" is required for secret retrieval but was not found in config."
78                    )
79                })?;
80
81            debug!(message = "Retrieving secrets from a backend.", backend = ?backend_name, keys = ?keys);
82            let backend_secrets = backend
83                .retrieve(keys.clone(), &mut signal_rx)
84                .map_err(|e| {
85                    format!("Error while retrieving secret from backend \"{backend_name}\": {e}.")
86                })
87                .await?;
88
89            for (k, v) in backend_secrets {
90                trace!(message = "Successfully retrieved a secret.", backend = ?backend_name, key = ?k);
91                secrets.insert(format!("{backend_name}.{k}"), v);
92            }
93        }
94
95        Ok(secrets)
96    }
97}
98
99impl Default for SecretBackendLoader {
100    /// Creates a new SecretBackendLoader with default settings.
101    /// By default, environment variable interpolation is enabled.
102    fn default() -> Self {
103        Self {
104            backends: IndexMap::new(),
105            secret_keys: HashMap::new(),
106            interpolate_env: true,
107        }
108    }
109}
110
111impl Process for SecretBackendLoader {
112    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
113        let config_string = prepare_input(input, self.interpolate_env)?;
114        // Collect secret placeholders just after env var processing
115        collect_secret_keys(&config_string, &mut self.secret_keys);
116        Ok(config_string)
117    }
118
119    fn merge(&mut self, table: Table, _: Option<ComponentHint>) -> Result<(), Vec<String>> {
120        if table.contains_key("secret") {
121            let additional = deserialize_table::<SecretBackendOuter>(table)?;
122            self.backends.extend(additional.secret);
123        }
124        Ok(())
125    }
126}
127
128impl Loader<SecretBackendLoader> for SecretBackendLoader {
129    fn take(self) -> SecretBackendLoader {
130        self
131    }
132}
133
134fn collect_secret_keys(input: &str, keys: &mut HashMap<String, HashSet<String>>) {
135    COLLECTOR.captures_iter(input).for_each(|cap| {
136        if let (Some(backend), Some(key)) = (cap.get(1), cap.get(2)) {
137            if let Some(keys) = keys.get_mut(backend.as_str()) {
138                keys.insert(key.as_str().to_string());
139            } else {
140                keys.insert(
141                    backend.as_str().to_string(),
142                    HashSet::from_iter(std::iter::once(key.as_str().to_string())),
143                );
144            }
145        }
146    });
147}
148
149pub fn interpolate(input: &str, secrets: &HashMap<String, String>) -> Result<String, Vec<String>> {
150    let mut errors = Vec::<String>::new();
151    let output = COLLECTOR
152        .replace_all(input, |caps: &Captures<'_>| {
153            caps.get(1)
154                .and_then(|b| caps.get(2).map(|k| (b, k)))
155                .and_then(|(b, k)| secrets.get(&format!("{}.{}", b.as_str(), k.as_str())))
156                .cloned()
157                .unwrap_or_else(|| {
158                    errors.push(format!(
159                        "Unable to find secret replacement for {}.",
160                        caps.get(0).unwrap().as_str()
161                    ));
162                    "".to_string()
163                })
164        })
165        .into_owned();
166    if errors.is_empty() {
167        Ok(output)
168    } else {
169        Err(errors)
170    }
171}
172
173#[cfg(test)]
174mod tests {
175    use std::collections::HashMap;
176
177    use indoc::indoc;
178
179    use super::{collect_secret_keys, interpolate};
180
181    #[test]
182    fn replacement() {
183        let secrets: HashMap<String, String> = vec![
184            ("a.secret.key".into(), "value".into()),
185            ("a...key".into(), "a...value".into()),
186            ("backend.path/to/secret".into(), "secret_value".into()),
187            ("backend.nested/dir/file".into(), "nested_value".into()),
188        ]
189        .into_iter()
190        .collect();
191
192        assert_eq!(
193            Ok("value".into()),
194            interpolate("SECRET[a.secret.key]", &secrets)
195        );
196        assert_eq!(
197            Ok("value value".into()),
198            interpolate("SECRET[a.secret.key] SECRET[a.secret.key]", &secrets)
199        );
200
201        assert_eq!(
202            Ok("xxxvalueyyy".into()),
203            interpolate("xxxSECRET[a.secret.key]yyy", &secrets)
204        );
205        assert_eq!(
206            Ok("a...value".into()),
207            interpolate("SECRET[a...key]", &secrets)
208        );
209        assert_eq!(
210            Ok("secret_value".into()),
211            interpolate("SECRET[backend.path/to/secret]", &secrets)
212        );
213        assert_eq!(
214            Ok("nested_value".into()),
215            interpolate("SECRET[backend.nested/dir/file]", &secrets)
216        );
217        assert_eq!(
218            Ok("xxxSECRET[non_matching_syntax]yyy".into()),
219            interpolate("xxxSECRET[non_matching_syntax]yyy", &secrets)
220        );
221        assert_eq!(
222            Err(vec![
223                "Unable to find secret replacement for SECRET[a.non.existing.key].".into()
224            ]),
225            interpolate("xxxSECRET[a.non.existing.key]yyy", &secrets)
226        );
227    }
228
229    #[test]
230    fn collection() {
231        let mut keys = HashMap::new();
232        collect_secret_keys(
233            indoc! {r"
234            SECRET[first_backend.secret_key]
235            SECRET[first_backend.secret-key]
236            SECRET[first_backend.another_secret_key]
237            SECRET[second_backend.secret_key]
238            SECRET[second_backend.secret.key]
239            SECRET[first_backend.a_third.secret_key]
240            SECRET[first_backend...an_extra_secret_key]
241            SECRET[first_backend.path/to/secret]
242            SECRET[second_backend.nested/dir/secret]
243            SECRET[non_matching_syntax]
244            SECRET[.non.matching.syntax]
245        "},
246            &mut keys,
247        );
248        assert_eq!(keys.len(), 2);
249        assert!(keys.contains_key("first_backend"));
250        assert!(keys.contains_key("second_backend"));
251
252        let first_backend_keys = keys.get("first_backend").unwrap();
253        assert_eq!(first_backend_keys.len(), 6);
254        assert!(first_backend_keys.contains("secret_key"));
255        assert!(first_backend_keys.contains("secret-key"));
256        assert!(first_backend_keys.contains("another_secret_key"));
257        assert!(first_backend_keys.contains("a_third.secret_key"));
258        assert!(first_backend_keys.contains("..an_extra_secret_key"));
259        assert!(first_backend_keys.contains("path/to/secret"));
260
261        let second_backend_keys = keys.get("second_backend").unwrap();
262        assert_eq!(second_backend_keys.len(), 3);
263        assert!(second_backend_keys.contains("secret_key"));
264        assert!(second_backend_keys.contains("secret.key"));
265        assert!(second_backend_keys.contains("nested/dir/secret"));
266    }
267
268    #[test]
269    fn collection_duplicates() {
270        let mut keys = HashMap::new();
271        collect_secret_keys(
272            indoc! {r"
273            SECRET[first_backend.secret_key]
274            SECRET[first_backend.secret_key]
275        "},
276            &mut keys,
277        );
278
279        let first_backend_keys = keys.get("first_backend").unwrap();
280        assert_eq!(first_backend_keys.len(), 1);
281        assert!(first_backend_keys.contains("secret_key"));
282    }
283}
vector/config/loading/secret.rs

vector/config/loading/
secret.rs
