vector/config/loading/
secret.rs

1use std::{
2    collections::{HashMap, HashSet},
3    io::Read,
4    sync::LazyLock,
5};
6
7use futures::TryFutureExt;
8use indexmap::IndexMap;
9use regex::{Captures, Regex};
10use serde::{Deserialize, Serialize};
11use vector_lib::config::ComponentKey;
12
13use crate::{
14    config::{
15        SecretBackend,
16        loading::{
17            ComponentHint, Loader, deserialize_config_map, prepare_input, process::Process,
18            representation::ConfigMap,
19        },
20    },
21    secrets::SecretBackends,
22    signal,
23};
24
25// The following regex aims to extract a pair of strings, the first being the secret backend name
26// and the second being the secret key. Here are some matching & non-matching examples:
27// - "SECRET[backend.secret_name]" will match and capture "backend" and "secret_name"
28// - "SECRET[my-backend.secret_name]" will match and capture "my-backend" and "secret_name"
29// - "SECRET[backend.secret.name]" will match and capture "backend" and "secret.name"
30// - "SECRET[backend..secret.name]" will match and capture "backend" and ".secret.name"
31// - "SECRET[backend.path/to/secret]" will match and capture "backend" and "path/to/secret"
32// - "SECRET[secret_name]" will not match
33// - "SECRET[.secret.name]" will not match
34pub static COLLECTOR: LazyLock<Regex> =
35    LazyLock::new(|| Regex::new(r"SECRET\[([[:word:]\-]+)\.([[:word:].\-/]+)\]").unwrap());
36
37/// Helper type for specifically deserializing secrets backends.
38#[derive(Debug, Default, Deserialize, Serialize)]
39pub(crate) struct SecretBackendOuter {
40    #[serde(default)]
41    pub(crate) secret: IndexMap<ComponentKey, SecretBackends>,
42}
43
44/// Loader for secrets backends.
45#[derive(Debug, Deserialize, Serialize)]
46pub struct SecretBackendLoader {
47    backends: IndexMap<ComponentKey, SecretBackends>,
48    secret_keys: HashMap<String, HashSet<String>>,
49    interpolate_env: bool,
50}
51
52impl SecretBackendLoader {
53    /// Sets whether to interpolate environment variables in the config.
54    pub const fn interpolate_env(mut self, interpolate: bool) -> Self {
55        self.interpolate_env = interpolate;
56        self
57    }
58
59    /// Retrieve secrets from backends.
60    /// Returns an empty HashMap if there are no secrets to retrieve.
61    pub(crate) async fn retrieve_secrets(
62        mut self,
63        signal_handler: &mut signal::SignalHandler,
64    ) -> Result<HashMap<String, String>, String> {
65        if self.secret_keys.is_empty() {
66            debug!(message = "No secret placeholder found, skipping secret resolution.");
67            return Ok(HashMap::new());
68        }
69
70        debug!(message = "Secret placeholders found, retrieving secrets from configured backends.");
71        let mut secrets: HashMap<String, String> = HashMap::new();
72        let mut signal_rx = signal_handler.subscribe();
73
74        for (backend_name, keys) in &self.secret_keys {
75            let backend = self
76                .backends
77                .get_mut(&ComponentKey::from(backend_name.clone()))
78                .ok_or_else(|| {
79                    format!(
80                        "Backend \"{backend_name}\" is required for secret retrieval but was not found in config."
81                    )
82                })?;
83
84            debug!(message = "Retrieving secrets from a backend.", backend = ?backend_name, keys = ?keys);
85            let backend_secrets = backend
86                .retrieve(keys.clone(), &mut signal_rx)
87                .map_err(|e| {
88                    format!("Error while retrieving secret from backend \"{backend_name}\": {e}.")
89                })
90                .await?;
91
92            for (k, v) in backend_secrets {
93                trace!(message = "Successfully retrieved a secret.", backend = ?backend_name, key = ?k);
94                secrets.insert(format!("{backend_name}.{k}"), v);
95            }
96        }
97
98        Ok(secrets)
99    }
100}
101
102impl Default for SecretBackendLoader {
103    fn default() -> Self {
104        Self {
105            backends: IndexMap::new(),
106            secret_keys: HashMap::new(),
107            interpolate_env: super::env_var_interpolation_enabled(),
108        }
109    }
110}
111
112impl Process for SecretBackendLoader {
113    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
114        let config_string = prepare_input(input, self.interpolate_env)?;
115        // Collect secret placeholders just after env var processing
116        collect_secret_keys(&config_string, &mut self.secret_keys);
117        Ok(config_string)
118    }
119
120    fn merge(&mut self, map: ConfigMap, _: Option<ComponentHint>) -> Result<(), Vec<String>> {
121        if map.contains_key("secret") {
122            let additional = deserialize_config_map::<SecretBackendOuter>(map)?;
123            self.backends.extend(additional.secret);
124        }
125        Ok(())
126    }
127}
128
129impl Loader<SecretBackendLoader> for SecretBackendLoader {
130    fn take(self) -> SecretBackendLoader {
131        self
132    }
133}
134
135fn collect_secret_keys(input: &str, keys: &mut HashMap<String, HashSet<String>>) {
136    COLLECTOR.captures_iter(input).for_each(|cap| {
137        if let (Some(backend), Some(key)) = (cap.get(1), cap.get(2)) {
138            if let Some(keys) = keys.get_mut(backend.as_str()) {
139                keys.insert(key.as_str().to_string());
140            } else {
141                keys.insert(
142                    backend.as_str().to_string(),
143                    HashSet::from_iter(std::iter::once(key.as_str().to_string())),
144                );
145            }
146        }
147    });
148}
149
150pub fn interpolate(input: &str, secrets: &HashMap<String, String>) -> Result<String, Vec<String>> {
151    let mut errors = Vec::<String>::new();
152    let output = COLLECTOR
153        .replace_all(input, |caps: &Captures<'_>| {
154            caps.get(1)
155                .and_then(|b| caps.get(2).map(|k| (b, k)))
156                .and_then(|(b, k)| secrets.get(&format!("{}.{}", b.as_str(), k.as_str())))
157                .cloned()
158                .unwrap_or_else(|| {
159                    errors.push(format!(
160                        "Unable to find secret replacement for {}.",
161                        caps.get(0).unwrap().as_str()
162                    ));
163                    "".to_string()
164                })
165        })
166        .into_owned();
167    if errors.is_empty() {
168        Ok(output)
169    } else {
170        Err(errors)
171    }
172}
173
174#[cfg(test)]
175mod tests {
176    use std::collections::HashMap;
177
178    use indoc::indoc;
179
180    use super::{collect_secret_keys, interpolate};
181
182    #[test]
183    fn replacement() {
184        let secrets: HashMap<String, String> = vec![
185            ("a.secret.key".into(), "value".into()),
186            ("a...key".into(), "a...value".into()),
187            ("backend.path/to/secret".into(), "secret_value".into()),
188            ("backend.nested/dir/file".into(), "nested_value".into()),
189            ("my-backend.secret.key".into(), "hyphenated_value".into()),
190        ]
191        .into_iter()
192        .collect();
193
194        assert_eq!(
195            Ok("value".into()),
196            interpolate("SECRET[a.secret.key]", &secrets)
197        );
198        assert_eq!(
199            Ok("value value".into()),
200            interpolate("SECRET[a.secret.key] SECRET[a.secret.key]", &secrets)
201        );
202
203        assert_eq!(
204            Ok("xxxvalueyyy".into()),
205            interpolate("xxxSECRET[a.secret.key]yyy", &secrets)
206        );
207        assert_eq!(
208            Ok("a...value".into()),
209            interpolate("SECRET[a...key]", &secrets)
210        );
211        assert_eq!(
212            Ok("secret_value".into()),
213            interpolate("SECRET[backend.path/to/secret]", &secrets)
214        );
215        assert_eq!(
216            Ok("nested_value".into()),
217            interpolate("SECRET[backend.nested/dir/file]", &secrets)
218        );
219        assert_eq!(
220            Ok("hyphenated_value".into()),
221            interpolate("SECRET[my-backend.secret.key]", &secrets)
222        );
223        assert_eq!(
224            Ok("xxxSECRET[non_matching_syntax]yyy".into()),
225            interpolate("xxxSECRET[non_matching_syntax]yyy", &secrets)
226        );
227        assert_eq!(
228            Err(vec![
229                "Unable to find secret replacement for SECRET[a.non.existing.key].".into()
230            ]),
231            interpolate("xxxSECRET[a.non.existing.key]yyy", &secrets)
232        );
233    }
234
235    #[test]
236    fn collection() {
237        let mut keys = HashMap::new();
238        collect_secret_keys(
239            indoc! {r"
240            SECRET[first_backend.secret_key]
241            SECRET[first_backend.secret-key]
242            SECRET[first_backend.another_secret_key]
243            SECRET[second_backend.secret_key]
244            SECRET[second_backend.secret.key]
245            SECRET[first_backend.a_third.secret_key]
246            SECRET[first_backend...an_extra_secret_key]
247            SECRET[third-backend.secret_key]
248            SECRET[first_backend.path/to/secret]
249            SECRET[second_backend.nested/dir/secret]
250            SECRET[third-backend.another-secret]
251            SECRET[non_matching_syntax]
252            SECRET[.non.matching.syntax]
253        "},
254            &mut keys,
255        );
256        assert_eq!(keys.len(), 3);
257        assert!(keys.contains_key("first_backend"));
258        assert!(keys.contains_key("second_backend"));
259        assert!(keys.contains_key("third-backend"));
260
261        let first_backend_keys = keys.get("first_backend").unwrap();
262        assert_eq!(first_backend_keys.len(), 6);
263        assert!(first_backend_keys.contains("secret_key"));
264        assert!(first_backend_keys.contains("secret-key"));
265        assert!(first_backend_keys.contains("another_secret_key"));
266        assert!(first_backend_keys.contains("a_third.secret_key"));
267        assert!(first_backend_keys.contains("..an_extra_secret_key"));
268        assert!(first_backend_keys.contains("path/to/secret"));
269
270        let second_backend_keys = keys.get("second_backend").unwrap();
271        assert_eq!(second_backend_keys.len(), 3);
272        assert!(second_backend_keys.contains("secret_key"));
273        assert!(second_backend_keys.contains("secret.key"));
274        assert!(second_backend_keys.contains("nested/dir/secret"));
275
276        let third_backend_keys = keys.get("third-backend").unwrap();
277        assert_eq!(third_backend_keys.len(), 2);
278        assert!(third_backend_keys.contains("secret_key"));
279        assert!(third_backend_keys.contains("another-secret"));
280    }
281
282    #[test]
283    fn collection_duplicates() {
284        let mut keys = HashMap::new();
285        collect_secret_keys(
286            indoc! {r"
287            SECRET[first_backend.secret_key]
288            SECRET[first_backend.secret_key]
289        "},
290            &mut keys,
291        );
292
293        let first_backend_keys = keys.get("first_backend").unwrap();
294        assert_eq!(first_backend_keys.len(), 1);
295        assert!(first_backend_keys.contains("secret_key"));
296    }
297}
vector/config/loading/secret.rs

vector/config/loading/
secret.rs
